From ada9f8b7ebd422a6918b5abcc7d11659bcdce046 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 16 Apr 2024 15:40:34 +0300 Subject: [PATCH 01/71] added tests for save to custom table node --- .../TbSaveToCustomCassandraTableNode.java | 8 +- .../TbSaveToCustomCassandraTableNodeTest.java | 248 ++++++++++++++++++ 2 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index 4b957db9cd..1e55e9d320 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -173,7 +173,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { throw new IllegalStateException("Invalid message structure, it is not a JSON Object:" + data); } else { JsonObject dataAsObject = data.getAsJsonObject(); - BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(saveStmt.bind()); + BoundStatementBuilder stmtBuilder = getStmtBuilder(); AtomicInteger i = new AtomicInteger(0); fieldsMap.forEach((key, value) -> { if (key.equals(ENTITY_ID)) { @@ -198,7 +198,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } else if (dataKeyElement.isJsonObject()) { stmtBuilder.setString(i.get(), dataKeyElement.getAsJsonObject().toString()); } else { - throw new IllegalStateException("Message data key: '" + key + "' with value: '" + value + "' is not a JSON Object or JSON Primitive!"); + throw new IllegalStateException("Message data key: '" + key + "' with value: '" + dataKeyElement + "' is not a JSON Object or JSON Primitive!"); } } else { throw new RuntimeException("Message data doesn't contain key: " + "'" + key + "'!"); @@ -209,6 +209,10 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } } + BoundStatementBuilder getStmtBuilder() { + return new BoundStatementBuilder(saveStmt.bind()); + } + private TbResultSetFuture executeAsyncWrite(TbContext ctx, Statement statement) { return executeAsync(ctx, statement, defaultWriteLevel); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java new file mode 100644 index 0000000000..699af15fa3 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -0,0 +1,248 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.action; + +import com.datastax.oss.driver.api.core.DefaultConsistencyLevel; +import com.datastax.oss.driver.api.core.ProtocolVersion; +import com.datastax.oss.driver.api.core.cql.BoundStatement; +import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; +import com.datastax.oss.driver.api.core.cql.ColumnDefinitions; +import com.datastax.oss.driver.api.core.cql.PreparedStatement; +import com.datastax.oss.driver.api.core.metadata.Node; +import com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry; + +import com.google.common.util.concurrent.SettableFuture; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ListeningExecutor; +import org.thingsboard.rule.engine.TestDbCallbackExecutor; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cassandra.guava.GuavaSession; +import org.thingsboard.server.dao.nosql.CassandraStatementTask; +import org.thingsboard.server.dao.nosql.TbResultSet; +import org.thingsboard.server.dao.nosql.TbResultSetFuture; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static java.util.Collections.emptyMap; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class TbSaveToCustomCassandraTableNodeTest { + + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("ac4ca02e-2ae6-404a-8f7e-c4ae31c56aa7")); + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("64ad971e-9cfa-49e4-9f59-faa1a2350c6e")); + + private final ListeningExecutor dbCallbackExecutor = new TestDbCallbackExecutor(); + + private TbSaveToCustomCassandraTableNode node; + private TbSaveToCustomCassandraTableNodeConfiguration config; + + @Mock + private TbContext ctxMock; + @Mock + private CassandraCluster cassandraClusterMock; + @Mock + private GuavaSession sessionMock; + @Mock + private PreparedStatement preparedStatementMock; + @Mock + private BoundStatement boundStatementMock; + @Mock + private BoundStatementBuilder boundStatementBuilderMock; + @Mock + private ColumnDefinitions columnDefinitionsMock; + @Mock + private CodecRegistry codecRegistryMock; + @Mock + private ProtocolVersion protocolVersionMock; + @Mock + private Node nodeMock; + + @Test + void givenCassandraClusterIsMissing_whenInit_thenThrowsException() { + node = new TbSaveToCustomCassandraTableNode(); + config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); + var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); + assertThatThrownBy(() -> node.init(ctxMock, configuration)) + .isInstanceOf(RuntimeException.class) + .hasMessage("Unable to connect to Cassandra database"); + } + + @Test + void givenFieldsMapIsEmpty_whenInit_thenThrowsException() { + node = new TbSaveToCustomCassandraTableNode(); + config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); + config.setFieldsMapping(emptyMap()); + var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); + + when(ctxMock.getCassandraCluster()).thenReturn(cassandraClusterMock); + + assertThatThrownBy(() -> node.init(ctxMock, configuration)) + .isInstanceOf(RuntimeException.class) + .hasMessage("Fields(key,value) map is empty!"); + } + + @Test + void givenInvalidMessageStructure_whenOnMsg_thenThrowsException() throws TbNodeException { + node = new TbSaveToCustomCassandraTableNode(); + config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); + config.setTableName("temperature_sensor"); + config.setFieldsMapping(Map.of("temp", "temperature")); + var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); + + initializeMocks(); + + node.init(ctxMock, configuration); + + String data = ""; + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data); + + assertThatThrownBy(() -> node.onMsg(ctxMock, msg)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Invalid message structure, it is not a JSON Object:" + null); + } + + @Test + void givenDataKeyIsMissingInMsg_whenOnMsg_thenThrowsException() throws TbNodeException { + node = new TbSaveToCustomCassandraTableNode(); + config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); + config.setTableName("temperature_sensor"); + config.setFieldsMapping(Map.of("temp", "temperature")); + var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); + + initializeMocks(); + when(preparedStatementMock.bind()).thenReturn(boundStatementMock); + when(boundStatementMock.getPreparedStatement()).thenReturn(preparedStatementMock); + when(preparedStatementMock.getVariableDefinitions()).thenReturn(columnDefinitionsMock); + when(boundStatementMock.codecRegistry()).thenReturn(codecRegistryMock); + when(boundStatementMock.protocolVersion()).thenReturn(protocolVersionMock); + when(boundStatementMock.getNode()).thenReturn(nodeMock); + + node.init(ctxMock, configuration); + + String data = """ + { + "humidity": 77 + } + """; + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data);; + + assertThatThrownBy(() -> node.onMsg(ctxMock, msg)) + .isInstanceOf(RuntimeException.class) + .hasMessage("Message data doesn't contain key: 'temp'!"); + } + + @Test + void givenUnsupportedData_whenOnMsg_thenThrowsException() throws TbNodeException { + node = new TbSaveToCustomCassandraTableNode(); + config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); + config.setTableName("temperature_sensor"); + config.setFieldsMapping(Map.of("temp", "temperature")); + var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); + + initializeMocks(); + when(preparedStatementMock.bind()).thenReturn(boundStatementMock); + when(boundStatementMock.getPreparedStatement()).thenReturn(preparedStatementMock); + when(preparedStatementMock.getVariableDefinitions()).thenReturn(columnDefinitionsMock); + when(boundStatementMock.getValues()).thenReturn(List.of(ByteBuffer.allocate(1))); + when(boundStatementMock.codecRegistry()).thenReturn(codecRegistryMock); + when(boundStatementMock.protocolVersion()).thenReturn(protocolVersionMock); + when(boundStatementMock.getNode()).thenReturn(nodeMock); + + node.init(ctxMock, configuration); + + String data = """ + { + "temp": [value] + } + """; + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data); + + + assertThatThrownBy(() -> node.onMsg(ctxMock, msg)) + .isInstanceOf(RuntimeException.class) + .hasMessage("Message data key: 'temp' with value: '[\"value\"]' is not a JSON Object or JSON Primitive!"); + } + + @Test + void givenValidMsgStructure_whenOnMsg_thenOk() throws Exception { + node = spy(new TbSaveToCustomCassandraTableNode()); + config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); + config.setTableName("temperature_sensor"); + config.setFieldsMapping(Map.of("temp", "temperature")); + var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); + + initializeMocks(); + doAnswer(invocation -> boundStatementBuilderMock).when(node).getStmtBuilder(); + when(boundStatementBuilderMock.setLong(anyInt(), anyLong())).thenReturn(boundStatementBuilderMock); + when(boundStatementBuilderMock.build()).thenReturn(boundStatementMock); + when(ctxMock.getTenantId()).thenReturn(TENANT_ID); + doAnswer(invocation -> { + SettableFuture mainFuture = SettableFuture.create(); + mainFuture.set(new TbResultSet(null, null, null)); + TbResultSetFuture value = new TbResultSetFuture(mainFuture); + return value; + }).when(ctxMock).submitCassandraWriteTask(any()); + when(ctxMock.getDbCallbackExecutor()).thenReturn(dbCallbackExecutor); + + node.init(ctxMock, configuration); + + String data = """ + { + "temp": 31, + "humidity": 77 + } + """; + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data); + + node.onMsg(ctxMock, msg); + + verify(sessionMock).prepare(eq("INSERT INTO cs_tb_temperature_sensor(temperature) VALUES(?)")); + verify(ctxMock).submitCassandraWriteTask(any(CassandraStatementTask.class)); + verify(ctxMock).tellSuccess(eq(msg)); + } + + private void initializeMocks() { + when(ctxMock.getCassandraCluster()).thenReturn(cassandraClusterMock); + when(cassandraClusterMock.getSession()).thenReturn(sessionMock); + when(cassandraClusterMock.getDefaultWriteConsistencyLevel()).thenReturn(DefaultConsistencyLevel.ONE); + when(sessionMock.prepare(anyString())).thenReturn(preparedStatementMock); + } +} From 00251f11b1dd871981f01b101d7cec34920d7cfe Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 29 May 2024 11:19:15 +0300 Subject: [PATCH 02/71] refactored tests --- .../TbSaveToCustomCassandraTableNodeTest.java | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index 699af15fa3..d760568bd4 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -25,6 +25,7 @@ import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry; import com.google.common.util.concurrent.SettableFuture; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -95,10 +96,14 @@ public class TbSaveToCustomCassandraTableNodeTest { @Mock private Node nodeMock; + @BeforeEach + void setUp() { + node = spy(new TbSaveToCustomCassandraTableNode()); + config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); + } + @Test void givenCassandraClusterIsMissing_whenInit_thenThrowsException() { - node = new TbSaveToCustomCassandraTableNode(); - config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); assertThatThrownBy(() -> node.init(ctxMock, configuration)) .isInstanceOf(RuntimeException.class) @@ -107,8 +112,6 @@ public class TbSaveToCustomCassandraTableNodeTest { @Test void givenFieldsMapIsEmpty_whenInit_thenThrowsException() { - node = new TbSaveToCustomCassandraTableNode(); - config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); config.setFieldsMapping(emptyMap()); var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); @@ -121,8 +124,6 @@ public class TbSaveToCustomCassandraTableNodeTest { @Test void givenInvalidMessageStructure_whenOnMsg_thenThrowsException() throws TbNodeException { - node = new TbSaveToCustomCassandraTableNode(); - config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); config.setTableName("temperature_sensor"); config.setFieldsMapping(Map.of("temp", "temperature")); var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); @@ -131,8 +132,7 @@ public class TbSaveToCustomCassandraTableNodeTest { node.init(ctxMock, configuration); - String data = ""; - TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data); + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); assertThatThrownBy(() -> node.onMsg(ctxMock, msg)) .isInstanceOf(IllegalStateException.class) @@ -141,8 +141,6 @@ public class TbSaveToCustomCassandraTableNodeTest { @Test void givenDataKeyIsMissingInMsg_whenOnMsg_thenThrowsException() throws TbNodeException { - node = new TbSaveToCustomCassandraTableNode(); - config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); config.setTableName("temperature_sensor"); config.setFieldsMapping(Map.of("temp", "temperature")); var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); @@ -171,8 +169,6 @@ public class TbSaveToCustomCassandraTableNodeTest { @Test void givenUnsupportedData_whenOnMsg_thenThrowsException() throws TbNodeException { - node = new TbSaveToCustomCassandraTableNode(); - config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); config.setTableName("temperature_sensor"); config.setFieldsMapping(Map.of("temp", "temperature")); var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); @@ -203,8 +199,6 @@ public class TbSaveToCustomCassandraTableNodeTest { @Test void givenValidMsgStructure_whenOnMsg_thenOk() throws Exception { - node = spy(new TbSaveToCustomCassandraTableNode()); - config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); config.setTableName("temperature_sensor"); config.setFieldsMapping(Map.of("temp", "temperature")); var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); From 6a9a933d4869475556d67d3fd3cad1526692bbeb Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 6 Jun 2024 16:18:43 +0300 Subject: [PATCH 03/71] added tests for check builder --- .../TbSaveToCustomCassandraTableNodeTest.java | 187 ++++++++++++------ 1 file changed, 131 insertions(+), 56 deletions(-) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index d760568bd4..9d0c646828 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -25,9 +25,14 @@ import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry; import com.google.common.util.concurrent.SettableFuture; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.JacksonUtil; @@ -47,14 +52,19 @@ import org.thingsboard.server.dao.nosql.CassandraStatementTask; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.nosql.TbResultSetFuture; -import java.nio.ByteBuffer; -import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.stream.Stream; import static java.util.Collections.emptyMap; -import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyDouble; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; @@ -102,6 +112,11 @@ public class TbSaveToCustomCassandraTableNodeTest { config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); } + @AfterEach + void tearDown() { + node.destroy(); + } + @Test void givenCassandraClusterIsMissing_whenInit_thenThrowsException() { var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); @@ -126,14 +141,12 @@ public class TbSaveToCustomCassandraTableNodeTest { void givenInvalidMessageStructure_whenOnMsg_thenThrowsException() throws TbNodeException { config.setTableName("temperature_sensor"); config.setFieldsMapping(Map.of("temp", "temperature")); - var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); - initializeMocks(); + mockOnInit(); - node.init(ctxMock, configuration); + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); - assertThatThrownBy(() -> node.onMsg(ctxMock, msg)) .isInstanceOf(IllegalStateException.class) .hasMessage("Invalid message structure, it is not a JSON Object:" + null); @@ -143,25 +156,18 @@ public class TbSaveToCustomCassandraTableNodeTest { void givenDataKeyIsMissingInMsg_whenOnMsg_thenThrowsException() throws TbNodeException { config.setTableName("temperature_sensor"); config.setFieldsMapping(Map.of("temp", "temperature")); - var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); - initializeMocks(); - when(preparedStatementMock.bind()).thenReturn(boundStatementMock); - when(boundStatementMock.getPreparedStatement()).thenReturn(preparedStatementMock); - when(preparedStatementMock.getVariableDefinitions()).thenReturn(columnDefinitionsMock); - when(boundStatementMock.codecRegistry()).thenReturn(codecRegistryMock); - when(boundStatementMock.protocolVersion()).thenReturn(protocolVersionMock); - when(boundStatementMock.getNode()).thenReturn(nodeMock); + mockOnInit(); + mockBoundStatement(); - node.init(ctxMock, configuration); + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); String data = """ { "humidity": 77 } """; - TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data);; - + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data); assertThatThrownBy(() -> node.onMsg(ctxMock, msg)) .isInstanceOf(RuntimeException.class) .hasMessage("Message data doesn't contain key: 'temp'!"); @@ -171,18 +177,11 @@ public class TbSaveToCustomCassandraTableNodeTest { void givenUnsupportedData_whenOnMsg_thenThrowsException() throws TbNodeException { config.setTableName("temperature_sensor"); config.setFieldsMapping(Map.of("temp", "temperature")); - var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); - initializeMocks(); - when(preparedStatementMock.bind()).thenReturn(boundStatementMock); - when(boundStatementMock.getPreparedStatement()).thenReturn(preparedStatementMock); - when(preparedStatementMock.getVariableDefinitions()).thenReturn(columnDefinitionsMock); - when(boundStatementMock.getValues()).thenReturn(List.of(ByteBuffer.allocate(1))); - when(boundStatementMock.codecRegistry()).thenReturn(codecRegistryMock); - when(boundStatementMock.protocolVersion()).thenReturn(protocolVersionMock); - when(boundStatementMock.getNode()).thenReturn(nodeMock); + mockOnInit(); + mockBoundStatement(); - node.init(ctxMock, configuration); + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); String data = """ { @@ -190,53 +189,129 @@ public class TbSaveToCustomCassandraTableNodeTest { } """; TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data); - - assertThatThrownBy(() -> node.onMsg(ctxMock, msg)) .isInstanceOf(RuntimeException.class) .hasMessage("Message data key: 'temp' with value: '[\"value\"]' is not a JSON Object or JSON Primitive!"); } @Test - void givenValidMsgStructure_whenOnMsg_thenOk() throws Exception { - config.setTableName("temperature_sensor"); - config.setFieldsMapping(Map.of("temp", "temperature")); - var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); + void givenValidMsgStructureAndKeyIsEntityId_whenOnMsg_thenOk() throws Exception { + config.setTableName("readings"); + config.setFieldsMapping(Map.of("$entityId", "entityId")); - initializeMocks(); - doAnswer(invocation -> boundStatementBuilderMock).when(node).getStmtBuilder(); - when(boundStatementBuilderMock.setLong(anyInt(), anyLong())).thenReturn(boundStatementBuilderMock); - when(boundStatementBuilderMock.build()).thenReturn(boundStatementMock); - when(ctxMock.getTenantId()).thenReturn(TENANT_ID); - doAnswer(invocation -> { - SettableFuture mainFuture = SettableFuture.create(); - mainFuture.set(new TbResultSet(null, null, null)); - TbResultSetFuture value = new TbResultSetFuture(mainFuture); - return value; - }).when(ctxMock).submitCassandraWriteTask(any()); - when(ctxMock.getDbCallbackExecutor()).thenReturn(dbCallbackExecutor); + mockOnInit(); + when(boundStatementBuilderMock.setUuid(anyInt(), any(UUID.class))).thenReturn(boundStatementBuilderMock); + when(boundStatementMock.getConsistencyLevel()).thenReturn(DefaultConsistencyLevel.ONE); + mockBoundStatementBuilder(); + mockSubmittingCassandraTask(); - node.init(ctxMock, configuration); + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); - String data = """ + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, TbMsg.EMPTY_JSON_OBJECT); + node.onMsg(ctxMock, msg); + + verify(sessionMock).prepare("INSERT INTO cs_tb_readings(entityId) VALUES(?)"); + verify(boundStatementBuilderMock).setUuid(anyInt(), eq(DEVICE_ID.getId())); + + ArgumentCaptor taskCaptor = ArgumentCaptor.forClass(CassandraStatementTask.class); + verify(ctxMock).submitCassandraWriteTask(taskCaptor.capture()); + CassandraStatementTask task = taskCaptor.getValue(); + assertThat(task.getTenantId()).isEqualTo(TENANT_ID); + assertThat(task.getSession()).isEqualTo(sessionMock); + assertThat(task.getStatement()).isEqualTo(boundStatementMock); + await().atMost(1, TimeUnit.SECONDS).untilAsserted( + () -> verify(ctxMock).tellSuccess(msg) + ); + } + + @ParameterizedTest + @MethodSource + void givenValidMsgStructure_whenOnMsg_thenSaveToCustomCassandraTable( + String valueFromData, + Consumer mockBuilderConsumer, + Consumer verifyBuilderConsumer) throws TbNodeException { + config.setTableName("sensor_readings"); + config.setFieldsMapping(Map.of("msgField1", "tableColumn1", "msgField2", "tableColumn2")); + + mockOnInit(); + mockBuilderConsumer.accept(boundStatementBuilderMock); + mockBoundStatementBuilder(); + mockSubmittingCassandraTask(); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + String data = String.format(""" { - "temp": 31, - "humidity": 77 + "msgField1": %s, + "msgField2": "value2", + "anotherField": 22 } - """; + """, valueFromData); TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data); - node.onMsg(ctxMock, msg); - verify(sessionMock).prepare(eq("INSERT INTO cs_tb_temperature_sensor(temperature) VALUES(?)")); - verify(ctxMock).submitCassandraWriteTask(any(CassandraStatementTask.class)); - verify(ctxMock).tellSuccess(eq(msg)); + verifyBuilderConsumer.accept(boundStatementBuilderMock); + + ArgumentCaptor taskCaptor = ArgumentCaptor.forClass(CassandraStatementTask.class); + verify(ctxMock).submitCassandraWriteTask(taskCaptor.capture()); + CassandraStatementTask task = taskCaptor.getValue(); + assertThat(task.getTenantId()).isEqualTo(TENANT_ID); + assertThat(task.getSession()).isEqualTo(sessionMock); + assertThat(task.getStatement()).isEqualTo(boundStatementMock); + await().atMost(1, TimeUnit.SECONDS).untilAsserted( + () -> verify(ctxMock).tellSuccess(msg) + ); + } + + private static Stream givenValidMsgStructure_whenOnMsg_thenSaveToCustomCassandraTable() { + return Stream.of( + Arguments.of("1.0", + (Consumer) mockBuilder -> when(mockBuilder.setDouble(anyInt(), anyDouble())).thenReturn(mockBuilder), + (Consumer) verifyBuilder -> verify(verifyBuilder).setDouble(anyInt(), eq(1.0))), + Arguments.of("2", + (Consumer) mockBuilder -> when(mockBuilder.setLong(anyInt(), anyLong())).thenReturn(mockBuilder), + (Consumer) verifyBuilder -> verify(verifyBuilder).setLong(anyInt(), eq(2L))), + Arguments.of("true", + (Consumer) mockBuilder -> when(mockBuilder.setBoolean(anyInt(), anyBoolean())).thenReturn(mockBuilder), + (Consumer) verifyBuilder -> verify(verifyBuilder).setBoolean(anyInt(), eq(true))), + Arguments.of("\"string value\"", + (Consumer) mockBuilder -> when(mockBuilder.setString(anyInt(), anyString())).thenReturn(mockBuilder), + (Consumer) verifyBuilder -> verify(verifyBuilder).setString(anyInt(), eq("string value"))), + Arguments.of("{\"key\":\"value\"}", + (Consumer) mockBuilder -> when(mockBuilder.setString(anyInt(), anyString())).thenReturn(mockBuilder), + (Consumer) verifyBuilder -> verify(verifyBuilder).setString(anyInt(), eq("{\"key\":\"value\"}"))) + ); } - private void initializeMocks() { + private void mockOnInit() { when(ctxMock.getCassandraCluster()).thenReturn(cassandraClusterMock); when(cassandraClusterMock.getSession()).thenReturn(sessionMock); when(cassandraClusterMock.getDefaultWriteConsistencyLevel()).thenReturn(DefaultConsistencyLevel.ONE); when(sessionMock.prepare(anyString())).thenReturn(preparedStatementMock); } + + private void mockBoundStatement() { + when(preparedStatementMock.bind()).thenReturn(boundStatementMock); + when(boundStatementMock.getPreparedStatement()).thenReturn(preparedStatementMock); + when(preparedStatementMock.getVariableDefinitions()).thenReturn(columnDefinitionsMock); + when(boundStatementMock.codecRegistry()).thenReturn(codecRegistryMock); + when(boundStatementMock.protocolVersion()).thenReturn(protocolVersionMock); + when(boundStatementMock.getNode()).thenReturn(nodeMock); + } + + private void mockBoundStatementBuilder() { + doAnswer(invocation -> boundStatementBuilderMock).when(node).getStmtBuilder(); + when(boundStatementBuilderMock.build()).thenReturn(boundStatementMock); + } + + private void mockSubmittingCassandraTask() { + when(ctxMock.getTenantId()).thenReturn(TENANT_ID); + doAnswer(invocation -> { + SettableFuture mainFuture = SettableFuture.create(); + mainFuture.set(new TbResultSet(null, null, null)); + return new TbResultSetFuture(mainFuture); + }).when(ctxMock).submitCassandraWriteTask(any()); + when(ctxMock.getDbCallbackExecutor()).thenReturn(dbCallbackExecutor); + } + } From 3368b9480b8ea6f49e8657209039cae8ae00f586 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 6 Jun 2024 16:29:38 +0300 Subject: [PATCH 04/71] added test for default config --- .../engine/action/TbSaveToCustomCassandraTableNodeTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index 9d0c646828..dab748b0a4 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -117,6 +117,12 @@ public class TbSaveToCustomCassandraTableNodeTest { node.destroy(); } + @Test + void verifyDefaultConfig() { + assertThat(config.getTableName()).isEqualTo(""); + assertThat(config.getFieldsMapping()).isEqualTo(Map.of("", "")); + } + @Test void givenCassandraClusterIsMissing_whenInit_thenThrowsException() { var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); From c137a144816c47446080925750ca5edcef80d020 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 7 Jun 2024 18:03:45 +0300 Subject: [PATCH 05/71] created complex test instead of parametrized --- .../TbSaveToCustomCassandraTableNode.java | 4 +- .../TbSaveToCustomCassandraTableNodeTest.java | 114 ++++++------------ 2 files changed, 38 insertions(+), 80 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index 1e55e9d320..d262edbb44 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -88,7 +88,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { config = TbNodeUtils.convert(configuration, TbSaveToCustomCassandraTableNodeConfiguration.class); cassandraCluster = ctx.getCassandraCluster(); if (cassandraCluster == null) { - throw new RuntimeException("Unable to connect to Cassandra database"); + throw new TbNodeException("Unable to connect to Cassandra database"); } else { startExecutor(); saveStmt = getSaveStmt(); @@ -170,7 +170,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { private ListenableFuture save(TbMsg msg, TbContext ctx) { JsonElement data = parser.parse(msg.getData()); if (!data.isJsonObject()) { - throw new IllegalStateException("Invalid message structure, it is not a JSON Object:" + data); + throw new IllegalStateException("Invalid message structure, it is not a JSON Object: " + data); } else { JsonObject dataAsObject = data.getAsJsonObject(); BoundStatementBuilder stmtBuilder = getStmtBuilder(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index dab748b0a4..d841928041 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -23,15 +23,11 @@ import com.datastax.oss.driver.api.core.cql.ColumnDefinitions; import com.datastax.oss.driver.api.core.cql.PreparedStatement; import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry; - import com.google.common.util.concurrent.SettableFuture; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -54,14 +50,10 @@ import org.thingsboard.server.dao.nosql.TbResultSetFuture; import java.util.Map; import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; -import java.util.stream.Stream; import static java.util.Collections.emptyMap; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyDouble; @@ -127,7 +119,7 @@ public class TbSaveToCustomCassandraTableNodeTest { void givenCassandraClusterIsMissing_whenInit_thenThrowsException() { var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); assertThatThrownBy(() -> node.init(ctxMock, configuration)) - .isInstanceOf(RuntimeException.class) + .isInstanceOf(TbNodeException.class) .hasMessage("Unable to connect to Cassandra database"); } @@ -155,7 +147,7 @@ public class TbSaveToCustomCassandraTableNodeTest { TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); assertThatThrownBy(() -> node.onMsg(ctxMock, msg)) .isInstanceOf(IllegalStateException.class) - .hasMessage("Invalid message structure, it is not a JSON Object:" + null); + .hasMessage("Invalid message structure, it is not a JSON Object: " + null); } @Test @@ -201,92 +193,44 @@ public class TbSaveToCustomCassandraTableNodeTest { } @Test - void givenValidMsgStructureAndKeyIsEntityId_whenOnMsg_thenOk() throws Exception { + void givenValidMsgStructure_whenOnMsg_thenSaveToCustomCassandraTable() throws TbNodeException { config.setTableName("readings"); - config.setFieldsMapping(Map.of("$entityId", "entityId")); - - mockOnInit(); - when(boundStatementBuilderMock.setUuid(anyInt(), any(UUID.class))).thenReturn(boundStatementBuilderMock); - when(boundStatementMock.getConsistencyLevel()).thenReturn(DefaultConsistencyLevel.ONE); - mockBoundStatementBuilder(); - mockSubmittingCassandraTask(); - - node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); - - TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, TbMsg.EMPTY_JSON_OBJECT); - node.onMsg(ctxMock, msg); - - verify(sessionMock).prepare("INSERT INTO cs_tb_readings(entityId) VALUES(?)"); - verify(boundStatementBuilderMock).setUuid(anyInt(), eq(DEVICE_ID.getId())); - - ArgumentCaptor taskCaptor = ArgumentCaptor.forClass(CassandraStatementTask.class); - verify(ctxMock).submitCassandraWriteTask(taskCaptor.capture()); - CassandraStatementTask task = taskCaptor.getValue(); - assertThat(task.getTenantId()).isEqualTo(TENANT_ID); - assertThat(task.getSession()).isEqualTo(sessionMock); - assertThat(task.getStatement()).isEqualTo(boundStatementMock); - await().atMost(1, TimeUnit.SECONDS).untilAsserted( - () -> verify(ctxMock).tellSuccess(msg) - ); - } - - @ParameterizedTest - @MethodSource - void givenValidMsgStructure_whenOnMsg_thenSaveToCustomCassandraTable( - String valueFromData, - Consumer mockBuilderConsumer, - Consumer verifyBuilderConsumer) throws TbNodeException { - config.setTableName("sensor_readings"); - config.setFieldsMapping(Map.of("msgField1", "tableColumn1", "msgField2", "tableColumn2")); + config.setFieldsMapping(Map.of( + "$entityId", "entityIdTableColumn", + "doubleField", "doubleTableColumn", + "longField", "longTableColumn", + "booleanField", "booleanTableColumn", + "stringField", "stringTableColumn", + "jsonField", "jsonTableColumn" + )); mockOnInit(); - mockBuilderConsumer.accept(boundStatementBuilderMock); mockBoundStatementBuilder(); mockSubmittingCassandraTask(); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); - - String data = String.format(""" + String data = """ { - "msgField1": %s, - "msgField2": "value2", - "anotherField": 22 + "doubleField": 22.5, + "longField": 56, + "booleanField": true, + "stringField": "some string", + "jsonField": { + "key": "value" + } } - """, valueFromData); + """; TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data); node.onMsg(ctxMock, msg); - verifyBuilderConsumer.accept(boundStatementBuilderMock); - + verifySettingStatementBuilder(); ArgumentCaptor taskCaptor = ArgumentCaptor.forClass(CassandraStatementTask.class); verify(ctxMock).submitCassandraWriteTask(taskCaptor.capture()); CassandraStatementTask task = taskCaptor.getValue(); assertThat(task.getTenantId()).isEqualTo(TENANT_ID); assertThat(task.getSession()).isEqualTo(sessionMock); assertThat(task.getStatement()).isEqualTo(boundStatementMock); - await().atMost(1, TimeUnit.SECONDS).untilAsserted( - () -> verify(ctxMock).tellSuccess(msg) - ); - } - - private static Stream givenValidMsgStructure_whenOnMsg_thenSaveToCustomCassandraTable() { - return Stream.of( - Arguments.of("1.0", - (Consumer) mockBuilder -> when(mockBuilder.setDouble(anyInt(), anyDouble())).thenReturn(mockBuilder), - (Consumer) verifyBuilder -> verify(verifyBuilder).setDouble(anyInt(), eq(1.0))), - Arguments.of("2", - (Consumer) mockBuilder -> when(mockBuilder.setLong(anyInt(), anyLong())).thenReturn(mockBuilder), - (Consumer) verifyBuilder -> verify(verifyBuilder).setLong(anyInt(), eq(2L))), - Arguments.of("true", - (Consumer) mockBuilder -> when(mockBuilder.setBoolean(anyInt(), anyBoolean())).thenReturn(mockBuilder), - (Consumer) verifyBuilder -> verify(verifyBuilder).setBoolean(anyInt(), eq(true))), - Arguments.of("\"string value\"", - (Consumer) mockBuilder -> when(mockBuilder.setString(anyInt(), anyString())).thenReturn(mockBuilder), - (Consumer) verifyBuilder -> verify(verifyBuilder).setString(anyInt(), eq("string value"))), - Arguments.of("{\"key\":\"value\"}", - (Consumer) mockBuilder -> when(mockBuilder.setString(anyInt(), anyString())).thenReturn(mockBuilder), - (Consumer) verifyBuilder -> verify(verifyBuilder).setString(anyInt(), eq("{\"key\":\"value\"}"))) - ); + verify(ctxMock).tellSuccess(msg); } private void mockOnInit() { @@ -307,6 +251,11 @@ public class TbSaveToCustomCassandraTableNodeTest { private void mockBoundStatementBuilder() { doAnswer(invocation -> boundStatementBuilderMock).when(node).getStmtBuilder(); + when(boundStatementBuilderMock.setUuid(anyInt(), any(UUID.class))).thenReturn(boundStatementBuilderMock); + when(boundStatementBuilderMock.setDouble(anyInt(), anyDouble())).thenReturn(boundStatementBuilderMock); + when(boundStatementBuilderMock.setLong(anyInt(), anyLong())).thenReturn(boundStatementBuilderMock); + when(boundStatementBuilderMock.setBoolean(anyInt(), anyBoolean())).thenReturn(boundStatementBuilderMock); + when(boundStatementBuilderMock.setString(anyInt(), anyString())).thenReturn(boundStatementBuilderMock); when(boundStatementBuilderMock.build()).thenReturn(boundStatementMock); } @@ -320,4 +269,13 @@ public class TbSaveToCustomCassandraTableNodeTest { when(ctxMock.getDbCallbackExecutor()).thenReturn(dbCallbackExecutor); } + private void verifySettingStatementBuilder() { + verify(boundStatementBuilderMock).setUuid(anyInt(), eq(DEVICE_ID.getId())); + verify(boundStatementBuilderMock).setDouble(anyInt(), eq(22.5)); + verify(boundStatementBuilderMock).setLong(anyInt(), eq(56L)); + verify(boundStatementBuilderMock).setBoolean(anyInt(), eq(true)); + verify(boundStatementBuilderMock).setString(anyInt(), eq("some string")); + verify(boundStatementBuilderMock).setString(anyInt(), eq("{\"key\":\"value\"}")); + } + } From e561e268719dc353b15d03c0c0caf9fccf2abeb2 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 11 Jun 2024 10:41:12 +0300 Subject: [PATCH 06/71] added possibility to set ttl via config --- .../TbSaveToCustomCassandraTableNode.java | 52 ++++++++++++++++--- ...CustomCassandraTableNodeConfiguration.java | 2 + 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index 1aad815dfa..9a3950a1c2 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -21,6 +21,8 @@ import com.datastax.oss.driver.api.core.cql.BoundStatement; import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.datastax.oss.driver.api.core.cql.PreparedStatement; import com.datastax.oss.driver.api.core.cql.Statement; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Function; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -28,6 +30,7 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.gson.JsonPrimitive; +import jakarta.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.TbContext; @@ -35,20 +38,23 @@ import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.dao.cassandra.CassandraCluster; import org.thingsboard.server.dao.cassandra.guava.GuavaSession; import org.thingsboard.server.dao.nosql.CassandraStatementTask; import org.thingsboard.server.dao.nosql.TbResultSetFuture; -import jakarta.annotation.Nullable; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import static org.thingsboard.common.util.DonAsynchron.withCallback; @@ -57,6 +63,7 @@ import static org.thingsboard.common.util.DonAsynchron.withCallback; @RuleNode(type = ComponentType.ACTION, name = "save to custom table", configClazz = TbSaveToCustomCassandraTableNodeConfiguration.class, + version = 1, nodeDescription = "Node stores data from incoming Message payload to the Cassandra database into the predefined custom table" + " that should have cs_tb_ prefix, to avoid the data insertion to the common TB tables.
" + "Note: rule node can be used only for Cassandra DB.", @@ -81,6 +88,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { private PreparedStatement saveStmt; private ExecutorService readResultsProcessingExecutor; private Map fieldsMap; + private long ttl; @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { @@ -88,15 +96,25 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { cassandraCluster = ctx.getCassandraCluster(); if (cassandraCluster == null) { throw new RuntimeException("Unable to connect to Cassandra database"); - } else { - startExecutor(); - saveStmt = getSaveStmt(); + } + ctx.addTenantProfileListener(this::onTenantProfileUpdate); + onTenantProfileUpdate(ctx.getTenantProfile()); + startExecutor(); + saveStmt = getSaveStmt(); + } + + void onTenantProfileUpdate(TenantProfile tenantProfile) { + DefaultTenantProfileConfiguration configuration = (DefaultTenantProfileConfiguration) tenantProfile.getProfileData().getConfiguration(); + long tenantProfileDefaultStorageTtl = TimeUnit.DAYS.toSeconds(configuration.getDefaultStorageTtlDays()); + ttl = config.getDefaultTTL(); + if (ttl == 0L) { + ttl = tenantProfileDefaultStorageTtl; } } @Override public void onMsg(TbContext ctx, TbMsg msg) { - withCallback(save(msg, ctx), aVoid -> ctx.tellSuccess(msg), e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); + withCallback(save(msg, ctx, ttl), aVoid -> ctx.tellSuccess(msg), e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); } @Override @@ -163,10 +181,13 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { query.append("?, "); } } + if (ttl > 0) { + query.append(" USING TTL ?"); + } return query.toString(); } - private ListenableFuture save(TbMsg msg, TbContext ctx) { + private ListenableFuture save(TbMsg msg, TbContext ctx, long ttl) { JsonElement data = JsonParser.parseString(msg.getData()); if (!data.isJsonObject()) { throw new IllegalStateException("Invalid message structure, it is not a JSON Object:" + data); @@ -204,6 +225,9 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } i.getAndIncrement(); }); + if (ttl > 0) { + stmtBuilder.setInt(i.get(), (int) ttl); + } return getFuture(executeAsyncWrite(ctx, stmtBuilder.build()), rs -> null); } } @@ -240,4 +264,20 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { }, readResultsProcessingExecutor); } + @Override + public TbPair upgrade(int fromVersion, JsonNode oldConfiguration) throws TbNodeException { + boolean hasChanges = false; + switch (fromVersion) { + case 0: + if (!oldConfiguration.has("defaultTTL")) { + hasChanges = true; + ((ObjectNode) oldConfiguration).put("defaultTTL", 0); + } + break; + default: + break; + } + return new TbPair<>(hasChanges, oldConfiguration); + } + } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java index 0a5f153192..8d3d78e8b9 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java @@ -27,11 +27,13 @@ public class TbSaveToCustomCassandraTableNodeConfiguration implements NodeConfig private String tableName; private Map fieldsMapping; + private long defaultTTL; @Override public TbSaveToCustomCassandraTableNodeConfiguration defaultConfiguration() { TbSaveToCustomCassandraTableNodeConfiguration configuration = new TbSaveToCustomCassandraTableNodeConfiguration(); + configuration.setDefaultTTL(0L); configuration.setTableName(""); Map map = new HashMap<>(); map.put("", ""); From bef6cfd339b711b9a5e23d5dd39d4c08c760d6e9 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 19 Jun 2024 16:15:08 +0300 Subject: [PATCH 07/71] refactored tests --- .../TbSaveToCustomCassandraTableNode.java | 9 ++-- .../engine/telemetry/TbMsgTimeseriesNode.java | 2 +- .../TbSaveToCustomCassandraTableNodeTest.java | 41 ++++++++++--------- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index 2df4225b0f..d22ffe928a 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -103,18 +103,17 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { saveStmt = getSaveStmt(); } - void onTenantProfileUpdate(TenantProfile tenantProfile) { + private void onTenantProfileUpdate(TenantProfile tenantProfile) { DefaultTenantProfileConfiguration configuration = (DefaultTenantProfileConfiguration) tenantProfile.getProfileData().getConfiguration(); - long tenantProfileDefaultStorageTtl = TimeUnit.DAYS.toSeconds(configuration.getDefaultStorageTtlDays()); ttl = config.getDefaultTTL(); if (ttl == 0L) { - ttl = tenantProfileDefaultStorageTtl; + ttl = TimeUnit.DAYS.toSeconds(configuration.getDefaultStorageTtlDays()); } } @Override public void onMsg(TbContext ctx, TbMsg msg) { - withCallback(save(msg, ctx, ttl), aVoid -> ctx.tellSuccess(msg), e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); + withCallback(save(msg, ctx), aVoid -> ctx.tellSuccess(msg), e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); } @Override @@ -187,7 +186,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { return query.toString(); } - private ListenableFuture save(TbMsg msg, TbContext ctx, long ttl) { + private ListenableFuture save(TbMsg msg, TbContext ctx) { JsonElement data = JsonParser.parseString(msg.getData()); if (!data.isJsonObject()) { throw new IllegalStateException("Invalid message structure, it is not a JSON Object: " + data); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 081804639a..97b2ac116f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -75,7 +75,7 @@ public class TbMsgTimeseriesNode implements TbNode { onTenantProfileUpdate(ctx.getTenantProfile()); } - void onTenantProfileUpdate(TenantProfile tenantProfile) { + private void onTenantProfileUpdate(TenantProfile tenantProfile) { DefaultTenantProfileConfiguration configuration = (DefaultTenantProfileConfiguration) tenantProfile.getProfileData().getConfiguration(); tenantProfileDefaultStorageTtl = TimeUnit.DAYS.toSeconds(configuration.getDefaultStorageTtlDays()); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index 3ab997238a..5d520d5ab9 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -57,6 +57,7 @@ import org.thingsboard.server.dao.nosql.CassandraStatementTask; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.nosql.TbResultSetFuture; +import java.util.LinkedHashMap; import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -75,10 +76,10 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.never; import static org.mockito.BDDMockito.spy; import static org.mockito.BDDMockito.then; import static org.mockito.BDDMockito.willAnswer; -import static org.mockito.BDDMockito.never; @ExtendWith(MockitoExtension.class) public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgradeTest { @@ -136,7 +137,9 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); assertThatThrownBy(() -> node.init(ctxMock, configuration)) .isInstanceOf(TbNodeException.class) - .hasMessage("Unable to connect to Cassandra database"); + .hasMessage("Unable to connect to Cassandra database") + .extracting(e -> ((TbNodeException) e).isUnrecoverable()) + .isEqualTo(false); } @Test @@ -238,15 +241,15 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad return Stream.of( Arguments.of(0, 0, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?)", (Consumer) builder -> { - then(builder).should(never()).setInt(anyInt(), eq(0)); + then(builder).should(never()).setInt(anyInt(), anyInt()); }), Arguments.of(0, 5, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?) USING TTL ?", (Consumer) builder -> { - then(builder).should().setInt(anyInt(), eq(432000)); + then(builder).should().setInt(1, 432000); }), Arguments.of(20, 1, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?) USING TTL ?", (Consumer) builder -> { - then(builder).should().setInt(anyInt(), eq(20)); + then(builder).should().setInt(1, 20); }) ); } @@ -255,14 +258,14 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad public void givenValidMsgStructure_whenOnMsg_thenSaveToCustomCassandraTable() throws TbNodeException { config.setDefaultTTL(25L); config.setTableName("readings"); - config.setFieldsMapping(Map.of( - "$entityId", "entityIdTableColumn", - "doubleField", "doubleTableColumn", - "longField", "longTableColumn", - "booleanField", "booleanTableColumn", - "stringField", "stringTableColumn", - "jsonField", "jsonTableColumn" - )); + Map mappings = new LinkedHashMap<>(); + mappings.put("$entityId", "entityIdTableColumn"); + mappings.put("doubleField", "doubleTableColumn"); + mappings.put("longField", "longTableColumn"); + mappings.put("booleanField", "booleanTableColumn"); + mappings.put("stringField", "stringTableColumn"); + mappings.put("jsonField", "jsonTableColumn"); + config.setFieldsMapping(mappings); mockOnInit(); mockBoundStatementBuilder(); @@ -350,12 +353,12 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad } private void verifySettingStatementBuilder() { - then(boundStatementBuilderMock).should().setUuid(anyInt(), eq(DEVICE_ID.getId())); - then(boundStatementBuilderMock).should().setDouble(anyInt(), eq(22.5)); - then(boundStatementBuilderMock).should().setLong(anyInt(), eq(56L)); - then(boundStatementBuilderMock).should().setBoolean(anyInt(), eq(true)); - then(boundStatementBuilderMock).should().setString(anyInt(), eq("some string")); - then(boundStatementBuilderMock).should().setString(anyInt(), eq("{\"key\":\"value\"}")); + then(boundStatementBuilderMock).should().setUuid(0, DEVICE_ID.getId()); + then(boundStatementBuilderMock).should().setDouble(1, 22.5); + then(boundStatementBuilderMock).should().setLong(2, 56L); + then(boundStatementBuilderMock).should().setBoolean(3, true); + then(boundStatementBuilderMock).should().setString(4, "some string"); + then(boundStatementBuilderMock).should().setString(5, "{\"key\":\"value\"}"); then(boundStatementBuilderMock).should().setInt(anyInt(), eq(25)); } From bfdea4d7e3acac12c9b314b697b44dda3e560b08 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 20 Jun 2024 10:34:20 +0300 Subject: [PATCH 08/71] made exception in init method unrecoverable --- .../rule/engine/action/TbSaveToCustomCassandraTableNode.java | 2 +- .../engine/action/TbSaveToCustomCassandraTableNodeTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index d22ffe928a..e1aa1f6a13 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -95,7 +95,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { config = TbNodeUtils.convert(configuration, TbSaveToCustomCassandraTableNodeConfiguration.class); cassandraCluster = ctx.getCassandraCluster(); if (cassandraCluster == null) { - throw new TbNodeException("Unable to connect to Cassandra database"); + throw new TbNodeException("Unable to connect to Cassandra database", true); } ctx.addTenantProfileListener(this::onTenantProfileUpdate); onTenantProfileUpdate(ctx.getTenantProfile()); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index 5d520d5ab9..27d69f2d66 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -139,7 +139,7 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad .isInstanceOf(TbNodeException.class) .hasMessage("Unable to connect to Cassandra database") .extracting(e -> ((TbNodeException) e).isUnrecoverable()) - .isEqualTo(false); + .isEqualTo(true); } @Test From 88a768f06732c46987fcd462ab8d8c0dc23c8581 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 21 Jun 2024 10:50:40 +0300 Subject: [PATCH 09/71] added verification for match values insertion into statement --- .../TbSaveToCustomCassandraTableNodeTest.java | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index 27d69f2d66..b3f3f2caa4 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -34,6 +34,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.AbstractRuleNodeUpgradeTest; @@ -57,7 +58,8 @@ import org.thingsboard.server.dao.nosql.CassandraStatementTask; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.nosql.TbResultSetFuture; -import java.util.LinkedHashMap; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -74,7 +76,6 @@ import static org.mockito.ArgumentMatchers.anyDouble; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.BDDMockito.given; import static org.mockito.BDDMockito.never; import static org.mockito.BDDMockito.spy; @@ -255,16 +256,17 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad } @Test - public void givenValidMsgStructure_whenOnMsg_thenSaveToCustomCassandraTable() throws TbNodeException { + public void givenValidMsgStructure_whenOnMsg_thenVerifyMatchOfValuesInsertionOrderIntoStatementAndSaveToCustomCassandraTable() throws TbNodeException { config.setDefaultTTL(25L); config.setTableName("readings"); - Map mappings = new LinkedHashMap<>(); - mappings.put("$entityId", "entityIdTableColumn"); - mappings.put("doubleField", "doubleTableColumn"); - mappings.put("longField", "longTableColumn"); - mappings.put("booleanField", "booleanTableColumn"); - mappings.put("stringField", "stringTableColumn"); - mappings.put("jsonField", "jsonTableColumn"); + Map mappings = Map.of( + "$entityId", "entityIdTableColumn", + "doubleField", "doubleTableColumn", + "longField", "longTableColumn", + "booleanField", "booleanTableColumn", + "stringField", "stringTableColumn", + "jsonField", "jsonTableColumn" + ); config.setFieldsMapping(mappings); mockOnInit(); @@ -353,13 +355,15 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad } private void verifySettingStatementBuilder() { - then(boundStatementBuilderMock).should().setUuid(0, DEVICE_ID.getId()); - then(boundStatementBuilderMock).should().setDouble(1, 22.5); - then(boundStatementBuilderMock).should().setLong(2, 56L); - then(boundStatementBuilderMock).should().setBoolean(3, true); - then(boundStatementBuilderMock).should().setString(4, "some string"); - then(boundStatementBuilderMock).should().setString(5, "{\"key\":\"value\"}"); - then(boundStatementBuilderMock).should().setInt(anyInt(), eq(25)); + Map fieldsMap = (Map) ReflectionTestUtils.getField(node, "fieldsMap"); + List values = new ArrayList<>(fieldsMap.values()); + then(boundStatementBuilderMock).should().setUuid(values.indexOf("entityIdTableColumn"), DEVICE_ID.getId()); + then(boundStatementBuilderMock).should().setDouble(values.indexOf("doubleTableColumn"), 22.5); + then(boundStatementBuilderMock).should().setLong(values.indexOf("longTableColumn"), 56L); + then(boundStatementBuilderMock).should().setBoolean(values.indexOf("booleanTableColumn"), true); + then(boundStatementBuilderMock).should().setString(values.indexOf("stringTableColumn"), "some string"); + then(boundStatementBuilderMock).should().setString(values.indexOf("jsonTableColumn"), "{\"key\":\"value\"}"); + then(boundStatementBuilderMock).should().setInt(values.size(), 25); } private TenantProfile getTenantProfileWithTtl(int ttlInDays) { From 4307d6f13ae2e8ca9887ebe95f468dd1f22d6828 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 1 Jul 2024 10:22:40 +0300 Subject: [PATCH 10/71] fixed rewriting ttl for existing nodes --- .../TbSaveToCustomCassandraTableNode.java | 28 +++++++--- ...CustomCassandraTableNodeConfiguration.java | 4 +- .../TbSaveToCustomCassandraTableNodeTest.java | 56 +++++++++++++++++-- 3 files changed, 73 insertions(+), 15 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index e1aa1f6a13..fcaf2420e9 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -21,6 +21,8 @@ import com.datastax.oss.driver.api.core.cql.BoundStatement; import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.datastax.oss.driver.api.core.cql.PreparedStatement; import com.datastax.oss.driver.api.core.cql.Statement; +import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; +import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Function; @@ -88,7 +90,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { private PreparedStatement saveStmt; private ExecutorService readResultsProcessingExecutor; private Map fieldsMap; - private long ttl; + private Integer ttl; @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { @@ -97,6 +99,9 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { if (cassandraCluster == null) { throw new TbNodeException("Unable to connect to Cassandra database", true); } + if (!isTableExists()) { + throw new TbNodeException("Table '" + TABLE_PREFIX + config.getTableName() + "' does not exist in Cassandra cluster.", true); + } ctx.addTenantProfileListener(this::onTenantProfileUpdate); onTenantProfileUpdate(ctx.getTenantProfile()); startExecutor(); @@ -106,8 +111,8 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { private void onTenantProfileUpdate(TenantProfile tenantProfile) { DefaultTenantProfileConfiguration configuration = (DefaultTenantProfileConfiguration) tenantProfile.getProfileData().getConfiguration(); ttl = config.getDefaultTTL(); - if (ttl == 0L) { - ttl = TimeUnit.DAYS.toSeconds(configuration.getDefaultStorageTtlDays()); + if (ttl != null && ttl == 0) { + ttl = (int) TimeUnit.DAYS.toSeconds(configuration.getDefaultStorageTtlDays()); } } @@ -132,6 +137,15 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } } + private boolean isTableExists() { + KeyspaceMetadata keyspaceMetadata = getSession().getMetadata().getKeyspace(cassandraCluster.getKeyspaceName()).orElse(null); + if (keyspaceMetadata != null) { + TableMetadata tableMetadata = keyspaceMetadata.getTable(TABLE_PREFIX + config.getTableName()).orElse(null); + return tableMetadata != null; + } + return false; + } + private PreparedStatement prepare(String query) { return getSession().prepare(query); } @@ -180,7 +194,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { query.append("?, "); } } - if (ttl > 0) { + if (ttl != null && ttl > 0) { query.append(" USING TTL ?"); } return query.toString(); @@ -224,8 +238,8 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } i.getAndIncrement(); }); - if (ttl > 0) { - stmtBuilder.setInt(i.get(), (int) ttl); + if (ttl != null && ttl > 0) { + stmtBuilder.setInt(i.get(), ttl); } return getFuture(executeAsyncWrite(ctx, stmtBuilder.build()), rs -> null); } @@ -274,7 +288,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { case 0: if (!oldConfiguration.has("defaultTTL")) { hasChanges = true; - ((ObjectNode) oldConfiguration).put("defaultTTL", 0); + ((ObjectNode) oldConfiguration).putNull("defaultTTL"); } break; default: diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java index 8d3d78e8b9..3efae275da 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java @@ -27,13 +27,13 @@ public class TbSaveToCustomCassandraTableNodeConfiguration implements NodeConfig private String tableName; private Map fieldsMapping; - private long defaultTTL; + private Integer defaultTTL; @Override public TbSaveToCustomCassandraTableNodeConfiguration defaultConfiguration() { TbSaveToCustomCassandraTableNodeConfiguration configuration = new TbSaveToCustomCassandraTableNodeConfiguration(); - configuration.setDefaultTTL(0L); + configuration.setDefaultTTL(0); configuration.setTableName(""); Map map = new HashMap<>(); map.put("", ""); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index b3f3f2caa4..d691f1eaa4 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -21,7 +21,10 @@ import com.datastax.oss.driver.api.core.cql.BoundStatement; import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.datastax.oss.driver.api.core.cql.ColumnDefinitions; import com.datastax.oss.driver.api.core.cql.PreparedStatement; +import com.datastax.oss.driver.api.core.metadata.Metadata; import com.datastax.oss.driver.api.core.metadata.Node; +import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; +import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; import com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry; import com.google.common.util.concurrent.SettableFuture; import org.junit.jupiter.api.AfterEach; @@ -61,6 +64,7 @@ import org.thingsboard.server.dao.nosql.TbResultSetFuture; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -114,6 +118,12 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad private ProtocolVersion protocolVersionMock; @Mock private Node nodeMock; + @Mock + private Metadata metadataMock; + @Mock + private KeyspaceMetadata keyspaceMetadataMock; + @Mock + private TableMetadata tableMetadataMock; @BeforeEach public void setUp() { @@ -130,7 +140,7 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad public void verifyDefaultConfig() { assertThat(config.getTableName()).isEqualTo(""); assertThat(config.getFieldsMapping()).isEqualTo(Map.of("", "")); - assertThat(config.getDefaultTTL()).isEqualTo(0L); + assertThat(config.getDefaultTTL()).isEqualTo(0); } @Test @@ -143,12 +153,28 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad .isEqualTo(true); } + @Test + public void givenTableDoesNotExist_whenInit_thenThrowsException() { + config.setTableName("test_table"); + var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); + + mockCassandraCluster(); + given(keyspaceMetadataMock.getTable(anyString())).willReturn(Optional.empty()); + + assertThatThrownBy(() -> node.init(ctxMock, configuration)) + .isInstanceOf(TbNodeException.class) + .hasMessage("Table 'cs_tb_test_table' does not exist in Cassandra cluster.") + .extracting(e -> ((TbNodeException) e).isUnrecoverable()) + .isEqualTo(true); + } + @Test public void givenFieldsMapIsEmpty_whenInit_thenThrowsException() { + config.setTableName("test_table"); config.setFieldsMapping(emptyMap()); var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); - given(ctxMock.getCassandraCluster()).willReturn(cassandraClusterMock); + mockCassandraCluster(); given(ctxMock.getTenantProfile()).willReturn(getTenantProfileWithTtl(5)); assertThatThrownBy(() -> node.init(ctxMock, configuration)) @@ -215,7 +241,7 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad @ParameterizedTest @MethodSource - public void givenTtl_whenOnMsg_thenVerifyStatement(long ttlFromConfig, + public void givenTtl_whenOnMsg_thenVerifyStatement(Integer ttlFromConfig, int ttlFromTenantProfileInDays, String expectedQuery, Consumer verifyBuilder) throws TbNodeException { @@ -251,13 +277,17 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad Arguments.of(20, 1, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?) USING TTL ?", (Consumer) builder -> { then(builder).should().setInt(1, 20); + }), + Arguments.of(null, 2, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?)", + (Consumer) builder -> { + then(builder).should(never()).setInt(anyInt(), anyInt()); }) ); } @Test public void givenValidMsgStructure_whenOnMsg_thenVerifyMatchOfValuesInsertionOrderIntoStatementAndSaveToCustomCassandraTable() throws TbNodeException { - config.setDefaultTTL(25L); + config.setDefaultTTL(25); config.setTableName("readings"); Map mappings = Map.of( "$entityId", "entityIdTableColumn", @@ -311,19 +341,33 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad Arguments.of(0, "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"}}", true, + "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTTL\":null}" + ), + // default config for version 1 with upgrade from version 1 + Arguments.of(1, + "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTTL\":0}", + false, "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTTL\":0}" ) ); } private void mockOnInit() { - given(ctxMock.getCassandraCluster()).willReturn(cassandraClusterMock); + mockCassandraCluster(); given(ctxMock.getTenantProfile()).willReturn(getTenantProfileWithTtl(5)); - given(cassandraClusterMock.getSession()).willReturn(sessionMock); given(cassandraClusterMock.getDefaultWriteConsistencyLevel()).willReturn(DefaultConsistencyLevel.ONE); given(sessionMock.prepare(anyString())).willReturn(preparedStatementMock); } + private void mockCassandraCluster() { + given(ctxMock.getCassandraCluster()).willReturn(cassandraClusterMock); + given(cassandraClusterMock.getSession()).willReturn(sessionMock); + given(sessionMock.getMetadata()).willReturn(metadataMock); + given(cassandraClusterMock.getKeyspaceName()).willReturn("test_keyspace"); + given(metadataMock.getKeyspace(anyString())).willReturn(Optional.of(keyspaceMetadataMock)); + given(keyspaceMetadataMock.getTable(anyString())).willReturn(Optional.of(tableMetadataMock)); + } + private void mockBoundStatement() { given(preparedStatementMock.bind()).willReturn(boundStatementMock); given(boundStatementMock.getPreparedStatement()).willReturn(preparedStatementMock); From 47afd13cf7af100cb14d7ad72bc9aaaed6201ac3 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 1 Jul 2024 12:17:47 +0300 Subject: [PATCH 11/71] made exception on init recoverable --- .../action/TbSaveToCustomCassandraTableNode.java | 13 ++++--------- .../TbSaveToCustomCassandraTableNodeTest.java | 2 +- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index fcaf2420e9..ddc4357674 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -21,8 +21,6 @@ import com.datastax.oss.driver.api.core.cql.BoundStatement; import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.datastax.oss.driver.api.core.cql.PreparedStatement; import com.datastax.oss.driver.api.core.cql.Statement; -import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; -import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Function; @@ -100,7 +98,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { throw new TbNodeException("Unable to connect to Cassandra database", true); } if (!isTableExists()) { - throw new TbNodeException("Table '" + TABLE_PREFIX + config.getTableName() + "' does not exist in Cassandra cluster.", true); + throw new TbNodeException("Table '" + TABLE_PREFIX + config.getTableName() + "' does not exist in Cassandra cluster."); } ctx.addTenantProfileListener(this::onTenantProfileUpdate); onTenantProfileUpdate(ctx.getTenantProfile()); @@ -138,12 +136,9 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } private boolean isTableExists() { - KeyspaceMetadata keyspaceMetadata = getSession().getMetadata().getKeyspace(cassandraCluster.getKeyspaceName()).orElse(null); - if (keyspaceMetadata != null) { - TableMetadata tableMetadata = keyspaceMetadata.getTable(TABLE_PREFIX + config.getTableName()).orElse(null); - return tableMetadata != null; - } - return false; + var keyspaceMdOpt = getSession().getMetadata().getKeyspace(cassandraCluster.getKeyspaceName()); + return keyspaceMdOpt.map(keyspaceMetadata -> + keyspaceMetadata.getTable(TABLE_PREFIX + config.getTableName()).isPresent()).orElse(false); } private PreparedStatement prepare(String query) { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index d691f1eaa4..f3e7d0d004 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -165,7 +165,7 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad .isInstanceOf(TbNodeException.class) .hasMessage("Table 'cs_tb_test_table' does not exist in Cassandra cluster.") .extracting(e -> ((TbNodeException) e).isUnrecoverable()) - .isEqualTo(true); + .isEqualTo(false); } @Test From d3d20aa449b0ff7103a55bda971e66e0bc9a7d9f Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 18 Oct 2024 16:45:11 +0200 Subject: [PATCH 12/71] used enqueue for input node for avoid latency --- .../server/actors/ruleChain/DefaultTbContext.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index ec10402821..b5b389da78 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -172,8 +172,18 @@ class DefaultTbContext implements TbContext { @Override public void input(TbMsg msg, RuleChainId ruleChainId) { + ack(msg); + if (!msg.isValid()) { + return; + } msg.pushToStack(nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId()); - nodeCtx.getChainActor().tell(new RuleChainInputMsg(ruleChainId, msg)); + TbMsg inputMsg = msg.copyWithRuleChainId(ruleChainId); + TransportProtos.ToRuleEngineMsg toReMsg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(inputMsg)).build(); + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getQueueName(), getTenantId(), inputMsg.getOriginator()); + mainCtx.getClusterService().pushMsgToRuleEngine(tpi, inputMsg.getId(), toReMsg, null); } @Override From e2a356703f5c4b7e8909d182f210fb102cd1d246 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 18 Oct 2024 17:05:22 +0200 Subject: [PATCH 13/71] minor refactoring, ack used if push is success --- .../actors/ruleChain/DefaultTbContext.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index b5b389da78..23741ad965 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -109,6 +109,8 @@ import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.common.SimpleTbQueueCallback; import org.thingsboard.server.service.executors.PubSubRuleNodeExecutorProvider; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; @@ -172,18 +174,27 @@ class DefaultTbContext implements TbContext { @Override public void input(TbMsg msg, RuleChainId ruleChainId) { - ack(msg); if (!msg.isValid()) { return; } - msg.pushToStack(nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId()); TbMsg inputMsg = msg.copyWithRuleChainId(ruleChainId); + inputMsg.pushToStack(nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId()); TransportProtos.ToRuleEngineMsg toReMsg = TransportProtos.ToRuleEngineMsg.newBuilder() .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) .setTbMsg(TbMsg.toByteString(inputMsg)).build(); TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getQueueName(), getTenantId(), inputMsg.getOriginator()); - mainCtx.getClusterService().pushMsgToRuleEngine(tpi, inputMsg.getId(), toReMsg, null); + mainCtx.getClusterService().pushMsgToRuleEngine(tpi, inputMsg.getId(), toReMsg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + ack(msg); + } + + @Override + public void onFailure(Throwable error) { + tellFailure(msg, error); + } + }); } @Override From 726c95c661516115171bbf5e1a18e3fc05d60dce Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Sun, 20 Oct 2024 13:41:56 +0200 Subject: [PATCH 14/71] fixed tests --- .../flow/AbstractRuleEngineFlowIntegrationTest.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 9e36fed677..920ffdd211 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.rules.flow; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -59,6 +60,7 @@ import org.thingsboard.server.dao.event.EventService; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.mockito.Mockito.spy; @@ -331,6 +333,15 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule RuleChain finalRuleChain = rootRuleChain; RuleNode lastRuleNode = secondaryMetaData.getNodes().stream().filter(node -> !node.getId().equals(finalRuleChain.getFirstRuleNodeId())).findFirst().get(); + Awaitility.await().atMost(3, TimeUnit.SECONDS) + .until(() -> + getDebugEvents(savedTenant.getId(), lastRuleNode.getId(), 1000) + .getData() + .stream() + .filter(filterByPostTelemetryEventType()) + .count() == 2 + ); + eventsPage = getDebugEvents(savedTenant.getId(), lastRuleNode.getId(), 1000); events = eventsPage.getData().stream().filter(filterByPostTelemetryEventType()).collect(Collectors.toList()); From 3ecbd3258e4f39c7be88bc25e65c230daf7af94c Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Tue, 22 Oct 2024 12:38:57 +0300 Subject: [PATCH 15/71] Introduce kafka approach to work with edge-event instead of postgres --- .../service/edge/EdgeContextComponent.java | 136 ++- .../edge/rpc/AbstractEdgeGrpcSession.java | 815 +++++++++++++++ .../service/edge/rpc/EdgeGrpcService.java | 51 +- .../service/edge/rpc/EdgeGrpcSession.java | 937 +----------------- .../edge/rpc/KafkaEdgeEventService.java | 92 ++ .../edge/rpc/KafkaEdgeGrpcSession.java | 112 +++ .../edge/rpc/PostgresEdgeGrpcSession.java | 187 ++++ .../edge/rpc/processor/BaseEdgeProcessor.java | 356 +------ .../processor/alarm/AlarmEdgeProcessor.java | 67 +- .../processor/alarm/AlarmEdgeProcessorV1.java | 6 +- .../processor/alarm/BaseAlarmProcessor.java | 82 +- .../processor/asset/AssetEdgeProcessor.java | 36 +- .../rpc/processor/asset/AssetProcessor.java | 3 +- .../processor/asset/BaseAssetProcessor.java | 11 +- .../profile/AssetProfileEdgeProcessor.java | 36 +- .../asset/profile/AssetProfileProcessor.java | 4 +- .../profile/BaseAssetProfileProcessor.java | 12 +- .../customer/CustomerEdgeProcessor.java | 23 +- .../dashboard/BaseDashboardProcessor.java | 14 +- .../dashboard/DashboardEdgeProcessor.java | 34 +- .../dashboard/DashboardEdgeProcessorV2.java | 1 + .../processor/device/BaseDeviceProcessor.java | 19 +- .../processor/device/DeviceEdgeProcessor.java | 37 +- .../rpc/processor/device/DeviceProcessor.java | 2 +- .../profile/BaseDeviceProfileProcessor.java | 12 +- .../profile/DeviceProfileEdgeProcessor.java | 17 +- .../profile/DeviceProfileProcessor.java | 4 +- .../rpc/processor/edge/EdgeProcessor.java | 22 +- .../entityview/BaseEntityViewProcessor.java | 11 +- .../entityview/EntityViewEdgeProcessor.java | 31 +- .../NotificationEdgeProcessor.java | 17 + .../processor/oauth2/OAuth2EdgeProcessor.java | 29 +- .../ota/OtaPackageEdgeProcessor.java | 21 +- .../processor/queue/QueueEdgeProcessor.java | 21 +- .../relation/BaseRelationProcessor.java | 4 +- .../relation/RelationEdgeProcessor.java | 9 +- .../resource/BaseResourceProcessor.java | 11 +- .../resource/ResourceEdgeProcessor.java | 21 +- .../rule/RuleChainEdgeProcessor.java | 23 +- .../settings/AdminSettingsEdgeProcessor.java | 5 + .../telemetry/BaseTelemetryProcessor.java | 45 +- .../processor/tenant/TenantEdgeProcessor.java | 23 +- .../tenant/TenantProfileEdgeProcessor.java | 17 +- .../rpc/processor/user/UserEdgeProcessor.java | 38 +- .../widget/WidgetBundleEdgeProcessor.java | 23 +- .../widget/WidgetTypeEdgeProcessor.java | 21 +- .../queue/DefaultTbCoreConsumerService.java | 2 +- .../queue/DefaultTbEdgeConsumerService.java | 2 +- .../src/main/resources/thingsboard.yml | 2 +- .../server/edge/AbstractEdgeTest.java | 2 +- .../src/test/resources/logback-test.xml | 2 +- .../server/common/data/DataConstants.java | 1 + .../server/common/data/edge/EdgeEvent.java | 3 + .../thingsboard/edge/rpc/EdgeGrpcClient.java | 2 +- common/edge-api/src/main/proto/edge.proto | 5 + .../common/msg/edge/EdgeEventUpdateMsg.java | 3 + .../common/msg/edge/EdgeHighPriorityMsg.java | 5 + .../common/msg/edge/EdgeSessionMsg.java | 4 +- .../common/msg/edge/FromEdgeSyncResponse.java | 2 + .../common/msg/edge/ToEdgeSyncRequest.java | 2 + .../server/common/util/ProtoUtils.java | 38 + common/proto/src/main/proto/queue.proto | 15 +- .../server/queue/discovery/TopicService.java | 10 + .../provider/AwsSqsMonolithQueueFactory.java | 13 + .../provider/AwsSqsTbCoreQueueFactory.java | 13 + .../AwsSqsTbRuleEngineQueueFactory.java | 10 +- .../InMemoryMonolithQueueFactory.java | 12 + .../provider/KafkaMonolithQueueFactory.java | 26 + .../provider/KafkaTbCoreQueueFactory.java | 26 + .../KafkaTbRuleEngineQueueFactory.java | 17 +- .../provider/PubSubMonolithQueueFactory.java | 13 + .../provider/PubSubTbCoreQueueFactory.java | 13 + .../PubSubTbRuleEngineQueueFactory.java | 10 +- .../RabbitMqMonolithQueueFactory.java | 13 + .../provider/RabbitMqTbCoreQueueFactory.java | 13 + .../RabbitMqTbRuleEngineQueueFactory.java | 10 +- .../ServiceBusMonolithQueueFactory.java | 13 + .../ServiceBusTbCoreQueueFactory.java | 13 + .../ServiceBusTbRuleEngineQueueFactory.java | 10 +- .../queue/provider/TbCoreQueueFactory.java | 7 + .../provider/TbCoreQueueProducerProvider.java | 8 + .../provider/TbQueueProducerProvider.java | 3 + .../TbRuleEngineProducerProvider.java | 8 + .../provider/TbRuleEngineQueueFactory.java | 7 +- .../TbTransportQueueProducerProvider.java | 6 + .../TbVersionControlProducerProvider.java | 6 + .../server/dao/edge/BaseEdgeEventService.java | 6 +- .../DefaultEdgeSynchronizationManager.java | 5 +- 88 files changed, 2194 insertions(+), 1685 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java index f0f41e3083..79259c36b6 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java @@ -22,11 +22,14 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.cache.limits.RateLimitService; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; +import org.thingsboard.server.dao.alarm.AlarmCommentService; +import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.domain.DomainService; @@ -36,8 +39,10 @@ import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.notification.NotificationRuleService; import org.thingsboard.server.dao.notification.NotificationTargetService; import org.thingsboard.server.dao.notification.NotificationTemplateService; +import org.thingsboard.server.dao.oauth2.OAuth2ClientService; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.queue.QueueService; +import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.settings.AdminSettingsService; @@ -49,6 +54,8 @@ import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.rpc.EdgeEventStorageSettings; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; +import org.thingsboard.server.service.edge.rpc.constructor.asset.AssetMsgConstructorFactory; +import org.thingsboard.server.service.edge.rpc.constructor.device.DeviceMsgConstructorFactory; import org.thingsboard.server.service.edge.rpc.constructor.edge.EdgeMsgConstructor; import org.thingsboard.server.service.edge.rpc.processor.alarm.AlarmEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.alarm.AlarmEdgeProcessorFactory; @@ -83,149 +90,154 @@ import org.thingsboard.server.service.edge.rpc.processor.user.UserEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.widget.WidgetBundleEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.widget.WidgetTypeEdgeProcessor; import org.thingsboard.server.service.edge.rpc.sync.EdgeRequestsService; -import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.GrpcCallbackExecutorService; +@Lazy +@Data @Component @TbCoreComponent -@Data -@Lazy public class EdgeContextComponent { + // services @Autowired - private TbClusterService clusterService; + private AdminSettingsService adminSettingsService; @Autowired - private EdgeService edgeService; + private AlarmCommentService alarmCommentService; - @Autowired(required = false) - private EdgeRpcService edgeRpcService; + @Autowired + private AlarmService alarmService; @Autowired - private EdgeEventService edgeEventService; + private AssetProfileService assetProfileService; @Autowired - private AdminSettingsService adminSettingsService; + private AssetService assetService; @Autowired - private DeviceService deviceService; + private AttributesService attributesService; @Autowired - private AssetService assetService; + private CustomerService customerService; @Autowired - private EntityViewService entityViewService; + private DashboardService dashboardService; + + @Autowired + private DeviceCredentialsService deviceCredentialsService; @Autowired private DeviceProfileService deviceProfileService; @Autowired - private AssetProfileService assetProfileService; + private DeviceService deviceService; @Autowired - private AttributesService attributesService; + private DomainService domainService; @Autowired - private DashboardService dashboardService; + private EdgeEventService edgeEventService; @Autowired - private RuleChainService ruleChainService; + private EdgeRequestsService edgeRequestsService; @Autowired - private UserService userService; + private EdgeRpcService edgeRpcService; @Autowired - private CustomerService customerService; + private EdgeService edgeService; @Autowired - private WidgetTypeService widgetTypeService; + private EntityViewService entityViewService; @Autowired - private WidgetsBundleService widgetsBundleService; + private NotificationRuleService notificationRuleService; @Autowired - private EdgeRequestsService edgeRequestsService; + private NotificationTargetService notificationTargetService; @Autowired - private OtaPackageService otaPackageService; + private NotificationTemplateService notificationTemplateService; @Autowired - private TenantService tenantService; + private OAuth2ClientService oAuth2ClientService; @Autowired - private TenantProfileService tenantProfileService; + private OtaPackageService otaPackageService; @Autowired private QueueService queueService; @Autowired - private ResourceService resourceService; + private RateLimitService rateLimitService; @Autowired - private NotificationRuleService notificationRuleService; + private RelationService relationService; @Autowired - private NotificationTargetService notificationTargetService; + private ResourceService resourceService; @Autowired - private NotificationTemplateService notificationTemplateService; + private RuleChainService ruleChainService; @Autowired - private DomainService domainService; + private TbClusterService clusterService; @Autowired - private RateLimitService rateLimitService; + private TenantProfileService tenantProfileService; @Autowired - private NotificationRuleProcessor notificationRuleProcessor; + private TenantService tenantService; @Autowired - private AlarmEdgeProcessor alarmProcessor; + private UserService userService; @Autowired - private DeviceProfileEdgeProcessor deviceProfileProcessor; + private WidgetTypeService widgetTypeService; @Autowired - private AssetProfileEdgeProcessor assetProfileProcessor; + private WidgetsBundleService widgetsBundleService; + + // processors @Autowired - private EdgeProcessor edgeProcessor; + private AdminSettingsEdgeProcessor adminSettingsProcessor; @Autowired - private DeviceEdgeProcessor deviceProcessor; + private AlarmEdgeProcessor alarmProcessor; @Autowired private AssetEdgeProcessor assetProcessor; @Autowired - private EntityViewEdgeProcessor entityViewProcessor; + private AssetProfileEdgeProcessor assetProfileProcessor; @Autowired - private UserEdgeProcessor userProcessor; + private CustomerEdgeProcessor customerProcessor; @Autowired - private RelationEdgeProcessor relationProcessor; + private DashboardEdgeProcessor dashboardProcessor; @Autowired - private TelemetryEdgeProcessor telemetryProcessor; + private DeviceEdgeProcessor deviceProcessor; @Autowired - private DashboardEdgeProcessor dashboardProcessor; + private DeviceProfileEdgeProcessor deviceProfileProcessor; @Autowired - private RuleChainEdgeProcessor ruleChainProcessor; + private EdgeProcessor edgeProcessor; @Autowired - private CustomerEdgeProcessor customerProcessor; + private EntityViewEdgeProcessor entityViewProcessor; @Autowired - private WidgetBundleEdgeProcessor widgetBundleProcessor; + private NotificationEdgeProcessor notificationEdgeProcessor; @Autowired - private WidgetTypeEdgeProcessor widgetTypeProcessor; + private NotificationRuleProcessor notificationRuleProcessor; @Autowired - private AdminSettingsEdgeProcessor adminSettingsProcessor; + private OAuth2EdgeProcessor oAuth2EdgeProcessor; @Autowired private OtaPackageEdgeProcessor otaPackageProcessor; @@ -233,6 +245,18 @@ public class EdgeContextComponent { @Autowired private QueueEdgeProcessor queueProcessor; + @Autowired + private RelationEdgeProcessor relationProcessor; + + @Autowired + private ResourceEdgeProcessor resourceProcessor; + + @Autowired + private RuleChainEdgeProcessor ruleChainProcessor; + + @Autowired + private TelemetryEdgeProcessor telemetryProcessor; + @Autowired private TenantEdgeProcessor tenantProcessor; @@ -240,23 +264,28 @@ public class EdgeContextComponent { private TenantProfileEdgeProcessor tenantProfileProcessor; @Autowired - private ResourceEdgeProcessor resourceProcessor; + private UserEdgeProcessor userProcessor; @Autowired - private NotificationEdgeProcessor notificationEdgeProcessor; + private WidgetBundleEdgeProcessor widgetBundleProcessor; @Autowired - private OAuth2EdgeProcessor oAuth2EdgeProcessor; + private WidgetTypeEdgeProcessor widgetTypeProcessor; + // msg constructors @Autowired private EdgeMsgConstructor edgeMsgConstructor; + // factories @Autowired private AlarmEdgeProcessorFactory alarmEdgeProcessorFactory; @Autowired private AssetEdgeProcessorFactory assetEdgeProcessorFactory; + @Autowired + private AssetMsgConstructorFactory assetMsgConstructorFactory; + @Autowired private AssetProfileEdgeProcessorFactory assetProfileEdgeProcessorFactory; @@ -266,6 +295,9 @@ public class EdgeContextComponent { @Autowired private DeviceEdgeProcessorFactory deviceEdgeProcessorFactory; + @Autowired + private DeviceMsgConstructorFactory deviceMsgConstructorFactory; + @Autowired private DeviceProfileEdgeProcessorFactory deviceProfileEdgeProcessorFactory; @@ -278,12 +310,12 @@ public class EdgeContextComponent { @Autowired private ResourceEdgeProcessorFactory resourceEdgeProcessorFactory; + // config @Autowired private EdgeEventStorageSettings edgeEventStorageSettings; - @Autowired - private DbCallbackExecutorService dbCallbackExecutor; - + // callback @Autowired private GrpcCallbackExecutorService grpcCallbackExecutorService; + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java new file mode 100644 index 0000000000..f92daa588a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java @@ -0,0 +1,815 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import io.grpc.stub.StreamObserver; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.springframework.data.util.Pair; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EdgeUtils; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.limit.LimitedApi; +import org.thingsboard.server.common.data.notification.rule.trigger.EdgeCommunicationFailureTrigger; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; +import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; +import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; +import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; +import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; +import org.thingsboard.server.gen.edge.v1.AttributesRequestMsg; +import org.thingsboard.server.gen.edge.v1.ConnectRequestMsg; +import org.thingsboard.server.gen.edge.v1.ConnectResponseCode; +import org.thingsboard.server.gen.edge.v1.ConnectResponseMsg; +import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; +import org.thingsboard.server.gen.edge.v1.DeviceCredentialsRequestMsg; +import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; +import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; +import org.thingsboard.server.gen.edge.v1.DeviceRpcCallMsg; +import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; +import org.thingsboard.server.gen.edge.v1.DownlinkMsg; +import org.thingsboard.server.gen.edge.v1.DownlinkResponseMsg; +import org.thingsboard.server.gen.edge.v1.EdgeConfiguration; +import org.thingsboard.server.gen.edge.v1.EdgeUpdateMsg; +import org.thingsboard.server.gen.edge.v1.EdgeVersion; +import org.thingsboard.server.gen.edge.v1.EntityDataProto; +import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; +import org.thingsboard.server.gen.edge.v1.EntityViewsRequestMsg; +import org.thingsboard.server.gen.edge.v1.RelationRequestMsg; +import org.thingsboard.server.gen.edge.v1.RelationUpdateMsg; +import org.thingsboard.server.gen.edge.v1.RequestMsg; +import org.thingsboard.server.gen.edge.v1.RequestMsgType; +import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; +import org.thingsboard.server.gen.edge.v1.ResponseMsg; +import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg; +import org.thingsboard.server.gen.edge.v1.SyncCompletedMsg; +import org.thingsboard.server.gen.edge.v1.UplinkMsg; +import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; +import org.thingsboard.server.gen.edge.v1.UserCredentialsRequestMsg; +import org.thingsboard.server.gen.edge.v1.WidgetBundleTypesRequestMsg; +import org.thingsboard.server.service.edge.EdgeContextComponent; +import org.thingsboard.server.service.edge.rpc.fetch.EdgeEventFetcher; +import org.thingsboard.server.service.edge.rpc.processor.alarm.AlarmProcessor; +import org.thingsboard.server.service.edge.rpc.processor.asset.AssetProcessor; +import org.thingsboard.server.service.edge.rpc.processor.asset.profile.AssetProfileProcessor; +import org.thingsboard.server.service.edge.rpc.processor.dashboard.DashboardProcessor; +import org.thingsboard.server.service.edge.rpc.processor.device.DeviceProcessor; +import org.thingsboard.server.service.edge.rpc.processor.device.profile.DeviceProfileProcessor; +import org.thingsboard.server.service.edge.rpc.processor.entityview.EntityViewProcessor; +import org.thingsboard.server.service.edge.rpc.processor.relation.RelationProcessor; +import org.thingsboard.server.service.edge.rpc.processor.resource.ResourceProcessor; + +import java.io.Closeable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.BiConsumer; + +@Slf4j +@Data +public abstract class AbstractEdgeGrpcSession> implements EdgeGrpcSession, Closeable { + + protected static final ReentrantLock downlinkMsgLock = new ReentrantLock(); + protected static final ConcurrentLinkedQueue highPriorityQueue = new ConcurrentLinkedQueue<>(); + + protected static final int MAX_DOWNLINK_ATTEMPTS = 10; + protected static final String QUEUE_START_TS_ATTR_KEY = "queueStartTs"; + protected static final String QUEUE_START_SEQ_ID_ATTR_KEY = "queueStartSeqId"; + protected static final String RATE_LIMIT_REACHED = "Rate limit reached"; + + protected UUID sessionId; + protected BiConsumer sessionOpenListener; + protected BiConsumer sessionCloseListener; + + protected final EdgeSessionState sessionState = new EdgeSessionState(); + + protected EdgeContextComponent ctx; + protected Edge edge; + protected TenantId tenantId; + protected StreamObserver inputStream; + protected StreamObserver outputStream; + protected boolean connected; + protected volatile boolean syncCompleted; + + protected Long newStartTs; + protected Long previousStartTs; + protected Long newStartSeqId; + protected Long previousStartSeqId; + protected Long seqIdEnd; + + protected EdgeVersion edgeVersion; + protected int maxInboundMessageSize; + protected int clientMaxInboundMessageSize; + protected int maxHighPriorityQueueSizePerSession; + + protected ScheduledExecutorService sendDownlinkExecutorService; + + public AbstractEdgeGrpcSession(EdgeContextComponent ctx, StreamObserver outputStream, + BiConsumer sessionOpenListener, + BiConsumer sessionCloseListener, + ScheduledExecutorService sendDownlinkExecutorService, + int maxInboundMessageSize, int maxHighPriorityQueueSizePerSession) { + this.sessionId = UUID.randomUUID(); + this.ctx = ctx; + this.outputStream = outputStream; + this.sessionOpenListener = sessionOpenListener; + this.sessionCloseListener = sessionCloseListener; + this.sendDownlinkExecutorService = sendDownlinkExecutorService; + this.maxInboundMessageSize = maxInboundMessageSize; + this.maxHighPriorityQueueSizePerSession = maxHighPriorityQueueSizePerSession; + initInputStream(); + } + + protected abstract ListenableFuture processEdgeEvents() throws Exception; + + public void initInputStream() { + this.inputStream = new StreamObserver<>() { + @Override + public void onNext(RequestMsg requestMsg) { + if (!connected && requestMsg.getMsgType().equals(RequestMsgType.CONNECT_RPC_MESSAGE)) { + ConnectResponseMsg responseMsg = processConnect(requestMsg.getConnectRequestMsg()); + outputStream.onNext(ResponseMsg.newBuilder() + .setConnectResponseMsg(responseMsg) + .build()); + if (ConnectResponseCode.ACCEPTED != responseMsg.getResponseCode()) { + outputStream.onError(new RuntimeException(responseMsg.getErrorMsg())); + } else { + if (requestMsg.getConnectRequestMsg().hasMaxInboundMessageSize()) { + log.debug("[{}][{}] Client max inbound message size: {}", tenantId, sessionId, requestMsg.getConnectRequestMsg().getMaxInboundMessageSize()); + clientMaxInboundMessageSize = requestMsg.getConnectRequestMsg().getMaxInboundMessageSize(); + } + connected = true; + } + } + if (connected) { + if (requestMsg.getMsgType().equals(RequestMsgType.SYNC_REQUEST_RPC_MESSAGE)) { + if (requestMsg.hasSyncRequestMsg()) { + boolean fullSync = false; + if (requestMsg.getSyncRequestMsg().hasFullSync()) { + fullSync = requestMsg.getSyncRequestMsg().getFullSync(); + } + startSyncProcess(fullSync); + } else { + syncCompleted = true; + } + } + if (requestMsg.getMsgType().equals(RequestMsgType.UPLINK_RPC_MESSAGE)) { + if (requestMsg.hasUplinkMsg()) { + onUplinkMsg(requestMsg.getUplinkMsg()); + } + if (requestMsg.hasDownlinkResponseMsg()) { + onDownlinkResponse(requestMsg.getDownlinkResponseMsg()); + } + } + } + } + + @Override + public void onError(Throwable t) { + log.error("[{}][{}] Stream was terminated due to error:", tenantId, sessionId, t); + closeSession(); + } + + @Override + public void onCompleted() { + log.info("[{}][{}] Stream was closed and completed successfully!", tenantId, sessionId); + closeSession(); + } + + private void closeSession() { + connected = false; + if (edge != null) { + try { + sessionCloseListener.accept(edge, sessionId); + } catch (Exception ignored) { + } + } + try { + outputStream.onCompleted(); + } catch (Exception ignored) { + } + } + }; + } + + @Override + public void onConfigurationUpdate(Edge edge) { + log.debug("[{}] onConfigurationUpdate [{}]", this.sessionId, edge); + this.edge = edge; + this.tenantId = edge.getTenantId(); + EdgeUpdateMsg edgeConfig = EdgeUpdateMsg.newBuilder() + .setConfiguration(ctx.getEdgeMsgConstructor().constructEdgeConfiguration(edge)).build(); + ResponseMsg edgeConfigMsg = ResponseMsg.newBuilder() + .setEdgeUpdateMsg(edgeConfig) + .build(); + sendDownlinkMsg(edgeConfigMsg); + } + + protected void processEdgeEvents(EdgeEventFetcher fetcher, PageLink pageLink, SettableFuture> result) { + try { + if (!highPriorityQueue.isEmpty()) { + processHighPriorityEvents(); + } + PageData pageData = fetcher.fetchEdgeEvents(edge.getTenantId(), edge, pageLink); + if (isConnected() && !pageData.getData().isEmpty()) { + log.trace("[{}][{}][{}] event(s) are going to be processed.", this.tenantId, this.sessionId, pageData.getData().size()); + List downlinkMsgsPack = convertToDownlinkMsgsPack(pageData.getData()); + Futures.addCallback(sendDownlinkMsgsPack(downlinkMsgsPack), new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Boolean isInterrupted) { + if (Boolean.TRUE.equals(isInterrupted)) { + log.debug("[{}][{}][{}] Send downlink messages task was interrupted", tenantId, edge.getId(), sessionId); + result.set(null); + } else { + if (isConnected() && pageData.hasNext()) { + processEdgeEvents(fetcher, pageLink.nextPageLink(), result); + } else { + EdgeEvent latestEdgeEvent = pageData.getData().get(pageData.getData().size() - 1); + UUID idOffset = latestEdgeEvent.getUuidId(); + if (idOffset != null) { + Long newStartTs = Uuids.unixTimestamp(idOffset); + long newStartSeqId = latestEdgeEvent.getSeqId(); + result.set(Pair.of(newStartTs, newStartSeqId)); + } else { + result.set(null); + } + } + } + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}] Failed to send downlink msgs pack", sessionId, t); + result.setException(t); + } + }, ctx.getGrpcCallbackExecutorService()); + } else { + log.trace("[{}] no event(s) found. Stop processing edge events", this.sessionId); + result.set(null); + } + } catch (Exception e) { + log.error("[{}] Failed to fetch edge events", this.sessionId, e); + result.setException(e); + } + } + + private ConnectResponseMsg processConnect(ConnectRequestMsg request) { + log.trace("[{}] processConnect [{}]", this.sessionId, request); + Optional optional = ctx.getEdgeService().findEdgeByRoutingKey(TenantId.SYS_TENANT_ID, request.getEdgeRoutingKey()); + if (optional.isPresent()) { + edge = optional.get(); + this.tenantId = edge.getTenantId(); + try { + if (edge.getSecret().equals(request.getEdgeSecret())) { + sessionOpenListener.accept(edge.getId(), (T) this); + this.edgeVersion = request.getEdgeVersion(); + processSaveEdgeVersionAsAttribute(request.getEdgeVersion().name()); + return ConnectResponseMsg.newBuilder() + .setResponseCode(ConnectResponseCode.ACCEPTED) + .setErrorMsg("") + .setConfiguration(ctx.getEdgeMsgConstructor().constructEdgeConfiguration(edge)) + .setMaxInboundMessageSize(maxInboundMessageSize) + .build(); + } + String error = "Failed to validate the edge!"; + String failureMsg = String.format("%s Provided request secret: %s", error, request.getEdgeSecret()); + ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) + .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(error).build()); + return ConnectResponseMsg.newBuilder() + .setResponseCode(ConnectResponseCode.BAD_CREDENTIALS) + .setErrorMsg(failureMsg) + .setConfiguration(EdgeConfiguration.getDefaultInstance()).build(); + } catch (Exception e) { + String failureMsg = "Failed to process edge connection!"; + ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) + .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(e.getMessage()).build()); + log.error(failureMsg, e); + return ConnectResponseMsg.newBuilder() + .setResponseCode(ConnectResponseCode.SERVER_UNAVAILABLE) + .setErrorMsg(failureMsg) + .setConfiguration(EdgeConfiguration.getDefaultInstance()).build(); + } + } + return ConnectResponseMsg.newBuilder() + .setResponseCode(ConnectResponseCode.BAD_CREDENTIALS) + .setErrorMsg("Failed to find the edge! Routing key: " + request.getEdgeRoutingKey()) + .setConfiguration(EdgeConfiguration.getDefaultInstance()).build(); + } + + private void processSaveEdgeVersionAsAttribute(String edgeVersion) { + AttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(new StringDataEntry(DataConstants.EDGE_VERSION_ATTR_KEY, edgeVersion), System.currentTimeMillis()); + ctx.getAttributesService().save(this.tenantId, this.edge.getId(), AttributeScope.SERVER_SCOPE, attributeKvEntry); + } + + @Override + public void startSyncProcess(boolean fullSync) { + log.info("[{}][{}][{}] Staring edge sync process", this.tenantId, edge.getId(), this.sessionId); + syncCompleted = false; + interruptGeneralProcessingOnSync(); + doSync(new EdgeSyncCursor(ctx, edge, fullSync)); + } + + private void interruptGeneralProcessingOnSync() { + log.debug("[{}][{}][{}] Sync process started. General processing interrupted!", this.tenantId, edge.getId(), this.sessionId); + stopCurrentSendDownlinkMsgsTask(true); + } + + private void doSync(EdgeSyncCursor cursor) { + if (cursor.hasNext()) { + EdgeEventFetcher next = cursor.getNext(); + log.info("[{}][{}] starting sync process, cursor current idx = {}, class = {}", + this.tenantId, edge.getId(), cursor.getCurrentIdx(), next.getClass().getSimpleName()); + ListenableFuture> future = startProcessingEdgeEvents(next); + Futures.addCallback(future, new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Pair result) { + doSync(cursor); + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}][{}] Exception during sync process", tenantId, edge.getId(), t); + } + }, ctx.getGrpcCallbackExecutorService()); + } else { + log.info("[{}][{}] sync process completed", this.tenantId, edge.getId()); + DownlinkMsg syncCompleteDownlinkMsg = DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .setSyncCompletedMsg(SyncCompletedMsg.newBuilder().build()) + .build(); + Futures.addCallback(sendDownlinkMsgsPack(Collections.singletonList(syncCompleteDownlinkMsg)), new FutureCallback<>() { + @Override + public void onSuccess(Boolean isInterrupted) { + markSyncCompletedSendEdgeEventUpdate(); + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}][{}] Exception during sending sync complete", tenantId, edge.getId(), t); + markSyncCompletedSendEdgeEventUpdate(); + } + }, ctx.getGrpcCallbackExecutorService()); + } + } + + protected ListenableFuture sendDownlinkMsgsPack(List downlinkMsgsPack) { + interruptPreviousSendDownlinkMsgsTask(); + + sessionState.setSendDownlinkMsgsFuture(SettableFuture.create()); + sessionState.getPendingMsgsMap().clear(); + + downlinkMsgsPack.forEach(msg -> sessionState.getPendingMsgsMap().put(msg.getDownlinkMsgId(), msg)); + scheduleDownlinkMsgsPackSend(1); + + return sessionState.getSendDownlinkMsgsFuture(); + } + + private void interruptPreviousSendDownlinkMsgsTask() { + if (sessionState.getSendDownlinkMsgsFuture() != null && !sessionState.getSendDownlinkMsgsFuture().isDone() + || sessionState.getScheduledSendDownlinkTask() != null && !sessionState.getScheduledSendDownlinkTask().isCancelled()) { + log.debug("[{}][{}][{}] Previous send downlink future was not properly completed, stopping it now!", this.tenantId, edge.getId(), this.sessionId); + stopCurrentSendDownlinkMsgsTask(true); + } + } + + private void onUplinkMsg(UplinkMsg uplinkMsg) { + if (isRateLimitViolated(uplinkMsg)) { + return; + } + ListenableFuture> future = processUplinkMsg(uplinkMsg); + Futures.addCallback(future, new FutureCallback<>() { + @Override + public void onSuccess(@Nullable List result) { + sendResponseMessage(uplinkMsg.getUplinkMsgId(), true, null); + } + + @Override + public void onFailure(Throwable t) { + String errorMsg = EdgeUtils.createErrorMsgFromRootCauseAndStackTrace(t); + sendResponseMessage(uplinkMsg.getUplinkMsgId(), false, errorMsg); + } + }, ctx.getGrpcCallbackExecutorService()); + } + + private boolean isRateLimitViolated(UplinkMsg uplinkMsg) { + if (!ctx.getRateLimitService().checkRateLimit(LimitedApi.EDGE_UPLINK_MESSAGES, tenantId) || + !ctx.getRateLimitService().checkRateLimit(LimitedApi.EDGE_UPLINK_MESSAGES_PER_EDGE, tenantId, edge.getId())) { + String errorMsg = String.format("Failed to process uplink message. %s", RATE_LIMIT_REACHED); + sendResponseMessage(uplinkMsg.getUplinkMsgId(), false, errorMsg); + return true; + } + return false; + } + + private void scheduleDownlinkMsgsPackSend(int attempt) { + Runnable sendDownlinkMsgsTask = () -> { + try { + if (isConnected() && !sessionState.getPendingMsgsMap().values().isEmpty()) { + List copy = new ArrayList<>(sessionState.getPendingMsgsMap().values()); + if (attempt > 1) { + String error = "Failed to deliver the batch"; + String failureMsg = String.format("{%s}: {%s}", error, copy); + if (attempt == 2) { + // Send a failure notification only on the second attempt. + // This ensures that failure alerts are sent just once to avoid redundant notifications. + ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId) + .edgeId(edge.getId()).customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(error).build()); + } + log.warn("[{}][{}] {}, attempt: {}", this.tenantId, this.sessionId, failureMsg, attempt); + } + log.trace("[{}][{}][{}] downlink msg(s) are going to be send.", this.tenantId, this.sessionId, copy.size()); + for (DownlinkMsg downlinkMsg : copy) { + if (this.clientMaxInboundMessageSize != 0 && downlinkMsg.getSerializedSize() > this.clientMaxInboundMessageSize) { + String error = String.format("Client max inbound message size %s is exceeded. Please increase value of CLOUD_RPC_MAX_INBOUND_MESSAGE_SIZE " + + "env variable on the edge and restart it.", this.clientMaxInboundMessageSize); + String message = String.format("Downlink msg size %s exceeds client max inbound message size %s. " + + "Please increase value of CLOUD_RPC_MAX_INBOUND_MESSAGE_SIZE env variable on the edge and restart it.", downlinkMsg.getSerializedSize(), this.clientMaxInboundMessageSize); + log.error("[{}][{}][{}] {} Message {}", this.tenantId, edge.getId(), this.sessionId, message, downlinkMsg); + ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId) + .edgeId(edge.getId()).customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(message).error(error).build()); + sessionState.getPendingMsgsMap().remove(downlinkMsg.getDownlinkMsgId()); + } else { + sendDownlinkMsg(ResponseMsg.newBuilder() + .setDownlinkMsg(downlinkMsg) + .build()); + } + } + if (attempt < MAX_DOWNLINK_ATTEMPTS) { + scheduleDownlinkMsgsPackSend(attempt + 1); + } else { + String failureMsg = String.format("Failed to deliver messages: %s", copy); + log.warn("[{}][{}] Failed to deliver the batch after {} attempts. Next messages are going to be discarded {}", + this.tenantId, this.sessionId, MAX_DOWNLINK_ATTEMPTS, copy); + ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) + .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg) + .error("Failed to deliver messages after " + MAX_DOWNLINK_ATTEMPTS + " attempts").build()); + stopCurrentSendDownlinkMsgsTask(false); + } + } else { + stopCurrentSendDownlinkMsgsTask(false); + } + } catch (Exception e) { + log.warn("[{}][{}] Failed to send downlink msgs. Error msg {}", this.tenantId, this.sessionId, e.getMessage(), e); + stopCurrentSendDownlinkMsgsTask(true); + } + }; + + if (attempt == 1) { + sendDownlinkExecutorService.submit(sendDownlinkMsgsTask); + } else { + sessionState.setScheduledSendDownlinkTask( + sendDownlinkExecutorService.schedule( + sendDownlinkMsgsTask, + ctx.getEdgeEventStorageSettings().getSleepIntervalBetweenBatches(), + TimeUnit.MILLISECONDS) + ); + } + } + + private void sendResponseMessage(int uplinkMsgId, boolean success, String errorMsg) { + UplinkResponseMsg.Builder responseBuilder = UplinkResponseMsg.newBuilder() + .setUplinkMsgId(uplinkMsgId) + .setSuccess(success); + if (errorMsg != null) { + responseBuilder.setErrorMsg(errorMsg); + } + sendDownlinkMsg(ResponseMsg.newBuilder() + .setUplinkResponseMsg(responseBuilder.build()) + .build()); + } + + private void onDownlinkResponse(DownlinkResponseMsg msg) { + try { + if (msg.getSuccess()) { + sessionState.getPendingMsgsMap().remove(msg.getDownlinkMsgId()); + log.debug("[{}][{}] Msg has been processed successfully! Msg Id: [{}], Msg: {}", this.tenantId, edge.getRoutingKey(), msg.getDownlinkMsgId(), msg); + } else { + log.error("[{}][{}] Msg processing failed! Msg Id: [{}], Error msg: {}", this.tenantId, edge.getRoutingKey(), msg.getDownlinkMsgId(), msg.getErrorMsg()); + } + if (sessionState.getPendingMsgsMap().isEmpty()) { + log.debug("[{}][{}] Pending msgs map is empty. Stopping current iteration", this.tenantId, edge.getRoutingKey()); + stopCurrentSendDownlinkMsgsTask(false); + } + } catch (Exception e) { + log.error("[{}][{}] Can't process downlink response message [{}]", this.tenantId, this.sessionId, msg, e); + } + } + + protected void processHighPriorityEvents() { + try { + List highPriorityEvents = new ArrayList<>(); + EdgeEvent event; + while ((event = highPriorityQueue.poll()) != null) { + highPriorityEvents.add(event); + } + List downlinkMsgsPack = convertToDownlinkMsgsPack(highPriorityEvents); + sendDownlinkMsgsPack(downlinkMsgsPack).get(); + } catch (Exception e) { + log.error("[{}] Failed to process high priority events", this.sessionId, e); + } + } + + protected List convertToDownlinkMsgsPack(List edgeEvents) { + List result = new ArrayList<>(); + for (EdgeEvent edgeEvent : edgeEvents) { + log.trace("[{}][{}] converting edge event to downlink msg [{}]", this.tenantId, this.sessionId, edgeEvent); + DownlinkMsg downlinkMsg = null; + try { + switch (edgeEvent.getAction()) { + case UPDATED, ADDED, DELETED, ASSIGNED_TO_EDGE, UNASSIGNED_FROM_EDGE, ALARM_ACK, ALARM_CLEAR, + ALARM_DELETE, + CREDENTIALS_UPDATED, RELATION_ADD_OR_UPDATE, RELATION_DELETED, RPC_CALL, + ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER, ADDED_COMMENT, UPDATED_COMMENT, + DELETED_COMMENT -> { + downlinkMsg = convertEntityEventToDownlink(edgeEvent); + if (downlinkMsg != null && downlinkMsg.getWidgetTypeUpdateMsgCount() > 0) { + log.trace("[{}][{}] widgetTypeUpdateMsg message processed, downlinkMsgId = {}", this.tenantId, this.sessionId, downlinkMsg.getDownlinkMsgId()); + } else { + log.trace("[{}][{}] entity message processed [{}]", this.tenantId, this.sessionId, downlinkMsg); + } + } + case ATTRIBUTES_UPDATED, POST_ATTRIBUTES, ATTRIBUTES_DELETED, TIMESERIES_UPDATED -> + downlinkMsg = ctx.getTelemetryProcessor().convertTelemetryEventToDownlink(edge, edgeEvent); + default -> + log.warn("[{}][{}] Unsupported action type [{}]", this.tenantId, this.sessionId, edgeEvent.getAction()); + } + } catch (Exception e) { + log.error("[{}][{}] Exception during converting edge event to downlink msg", this.tenantId, this.sessionId, e); + } + if (downlinkMsg != null) { + result.add(downlinkMsg); + } + } + return result; + } + + protected ListenableFuture> startProcessingEdgeEvents(EdgeEventFetcher fetcher) { + SettableFuture> result = SettableFuture.create(); + PageLink pageLink = fetcher.getPageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount()); + processEdgeEvents(fetcher, pageLink, result); + return result; + } + + private void markSyncCompletedSendEdgeEventUpdate() { + syncCompleted = true; + ctx.getClusterService().onEdgeEventUpdate(new EdgeEventUpdateMsg(edge.getTenantId(), edge.getId())); + } + + private void stopCurrentSendDownlinkMsgsTask(Boolean isInterrupted) { + if (sessionState.getSendDownlinkMsgsFuture() != null && !sessionState.getSendDownlinkMsgsFuture().isDone()) { + sessionState.getSendDownlinkMsgsFuture().set(isInterrupted); + } + if (sessionState.getScheduledSendDownlinkTask() != null) { + sessionState.getScheduledSendDownlinkTask().cancel(true); + } + } + + private void sendDownlinkMsg(ResponseMsg downlinkMsg) { + if (downlinkMsg.getDownlinkMsg().getWidgetTypeUpdateMsgCount() > 0) { + log.trace("[{}][{}] Sending downlink widgetTypeUpdateMsg, downlinkMsgId = {}", this.tenantId, this.sessionId, downlinkMsg.getDownlinkMsg().getDownlinkMsgId()); + } else { + log.trace("[{}][{}] Sending downlink msg [{}]", this.tenantId, this.sessionId, downlinkMsg); + } + if (isConnected()) { + downlinkMsgLock.lock(); + try { + outputStream.onNext(downlinkMsg); + } catch (Exception e) { + log.error("[{}][{}] Failed to send downlink message [{}]", this.tenantId, this.sessionId, downlinkMsg, e); + connected = false; + sessionCloseListener.accept(edge, sessionId); + } finally { + downlinkMsgLock.unlock(); + } + log.trace("[{}][{}] Response msg successfully sent. downlinkMsgId = {}", this.tenantId, this.sessionId, downlinkMsg.getDownlinkMsg().getDownlinkMsgId()); + } + } + + protected DownlinkMsg convertEntityEventToDownlink(EdgeEvent edgeEvent) { + log.trace("[{}] Executing convertEntityEventToDownlink, edgeEvent [{}], action [{}]", edgeEvent.getTenantId(), edgeEvent, edgeEvent.getAction()); + return switch (edgeEvent.getType()) { + case EDGE -> ctx.getEdgeProcessor().convertEdgeEventToDownlink(edgeEvent); + case DEVICE -> ctx.getDeviceProcessor().convertDeviceEventToDownlink(edgeEvent, edgeVersion); + case DEVICE_PROFILE -> + ctx.getDeviceProfileProcessor().convertDeviceProfileEventToDownlink(edgeEvent, edgeVersion); + case ASSET_PROFILE -> + ctx.getAssetProfileProcessor().convertAssetProfileEventToDownlink(edgeEvent, edgeVersion); + case ASSET -> + ctx.getAssetProcessor().convertAssetEventToDownlink(edgeEvent, edgeVersion); + case ENTITY_VIEW -> ctx.getEntityViewProcessor().convertEntityViewEventToDownlink(edgeEvent, edgeVersion); + case DASHBOARD -> ctx.getDashboardProcessor().convertDashboardEventToDownlink(edgeEvent, edgeVersion); + case CUSTOMER -> ctx.getCustomerProcessor().convertCustomerEventToDownlink(edgeEvent, edgeVersion); + case RULE_CHAIN -> ctx.getRuleChainProcessor().convertRuleChainEventToDownlink(edgeEvent, edgeVersion); + case RULE_CHAIN_METADATA -> + ctx.getRuleChainProcessor().convertRuleChainMetadataEventToDownlink(edgeEvent, edgeVersion); + case ALARM -> ctx.getAlarmProcessor().convertAlarmEventToDownlink(edgeEvent, edgeVersion); + case ALARM_COMMENT -> ctx.getAlarmProcessor().convertAlarmCommentEventToDownlink(edgeEvent, edgeVersion); + case USER -> ctx.getUserProcessor().convertUserEventToDownlink(edgeEvent, edgeVersion); + case RELATION -> ctx.getRelationProcessor().convertRelationEventToDownlink(edgeEvent, edgeVersion); + case WIDGETS_BUNDLE -> + ctx.getWidgetBundleProcessor().convertWidgetsBundleEventToDownlink(edgeEvent, edgeVersion); + case WIDGET_TYPE -> ctx.getWidgetTypeProcessor().convertWidgetTypeEventToDownlink(edgeEvent, edgeVersion); + case ADMIN_SETTINGS -> + ctx.getAdminSettingsProcessor().convertAdminSettingsEventToDownlink(edgeEvent, edgeVersion); + case OTA_PACKAGE -> ctx.getOtaPackageProcessor().convertOtaPackageEventToDownlink(edgeEvent, edgeVersion); + case TB_RESOURCE -> ctx.getResourceProcessor().convertResourceEventToDownlink(edgeEvent, edgeVersion); + case QUEUE -> ctx.getQueueProcessor().convertQueueEventToDownlink(edgeEvent, edgeVersion); + case TENANT -> ctx.getTenantProcessor().convertTenantEventToDownlink(edgeEvent, edgeVersion); + case TENANT_PROFILE -> + ctx.getTenantProfileProcessor().convertTenantProfileEventToDownlink(edgeEvent, edgeVersion); + case NOTIFICATION_RULE -> ctx.getNotificationEdgeProcessor().convertNotificationRuleToDownlink(edgeEvent); + case NOTIFICATION_TARGET -> + ctx.getNotificationEdgeProcessor().convertNotificationTargetToDownlink(edgeEvent); + case NOTIFICATION_TEMPLATE -> + ctx.getNotificationEdgeProcessor().convertNotificationTemplateToDownlink(edgeEvent); + case OAUTH2_CLIENT -> + ctx.getOAuth2EdgeProcessor().convertOAuth2ClientEventToDownlink(edgeEvent, edgeVersion); + case DOMAIN -> ctx.getOAuth2EdgeProcessor().convertOAuth2DomainEventToDownlink(edgeEvent, edgeVersion); + default -> { + log.warn("[{}] Unsupported edge event type [{}]", edgeEvent.getTenantId(), edgeEvent); + yield null; + } + }; + } + + public void addEventToHighPriorityQueue(EdgeEvent edgeEvent) { + while (highPriorityQueue.size() > maxHighPriorityQueueSizePerSession) { + EdgeEvent oldestHighPriority = highPriorityQueue.poll(); + if (oldestHighPriority != null) { + log.warn("[{}][{}][{}] High priority queue is full. Removing oldest high priority event from queue {}", + this.tenantId, edge.getId(), this.sessionId, oldestHighPriority); + } + } + highPriorityQueue.add(edgeEvent); + } + + protected ListenableFuture> processUplinkMsg(UplinkMsg uplinkMsg) { + List> result = new ArrayList<>(); + try { + if (uplinkMsg.getDeviceProfileUpdateMsgCount() > 0) { + for (DeviceProfileUpdateMsg deviceProfileUpdateMsg : uplinkMsg.getDeviceProfileUpdateMsgList()) { + result.add(((DeviceProfileProcessor) ctx.getDeviceProfileEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processDeviceProfileMsgFromEdge(edge.getTenantId(), edge, deviceProfileUpdateMsg)); + } + } + if (uplinkMsg.getDeviceUpdateMsgCount() > 0) { + for (DeviceUpdateMsg deviceUpdateMsg : uplinkMsg.getDeviceUpdateMsgList()) { + result.add(((DeviceProcessor) ctx.getDeviceEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processDeviceMsgFromEdge(edge.getTenantId(), edge, deviceUpdateMsg)); + } + } + if (uplinkMsg.getDeviceCredentialsUpdateMsgCount() > 0) { + for (DeviceCredentialsUpdateMsg deviceCredentialsUpdateMsg : uplinkMsg.getDeviceCredentialsUpdateMsgList()) { + result.add(((DeviceProcessor) ctx.getDeviceEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processDeviceCredentialsMsgFromEdge(edge.getTenantId(), edge.getId(), deviceCredentialsUpdateMsg)); + } + } + if (uplinkMsg.getAssetProfileUpdateMsgCount() > 0) { + for (AssetProfileUpdateMsg assetProfileUpdateMsg : uplinkMsg.getAssetProfileUpdateMsgList()) { + result.add(((AssetProfileProcessor) ctx.getAssetProfileEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processAssetProfileMsgFromEdge(edge.getTenantId(), edge, assetProfileUpdateMsg)); + } + } + if (uplinkMsg.getAssetUpdateMsgCount() > 0) { + for (AssetUpdateMsg assetUpdateMsg : uplinkMsg.getAssetUpdateMsgList()) { + result.add(((AssetProcessor) ctx.getAssetEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processAssetMsgFromEdge(edge.getTenantId(), edge, assetUpdateMsg)); + } + } + if (uplinkMsg.getEntityViewUpdateMsgCount() > 0) { + for (EntityViewUpdateMsg entityViewUpdateMsg : uplinkMsg.getEntityViewUpdateMsgList()) { + result.add(((EntityViewProcessor) ctx.getEntityViewProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processEntityViewMsgFromEdge(edge.getTenantId(), edge, entityViewUpdateMsg)); + } + } + if (uplinkMsg.getEntityDataCount() > 0) { + for (EntityDataProto entityData : uplinkMsg.getEntityDataList()) { + result.addAll(ctx.getTelemetryProcessor().processTelemetryMsg(edge.getTenantId(), entityData)); + } + } + if (uplinkMsg.getAlarmUpdateMsgCount() > 0) { + for (AlarmUpdateMsg alarmUpdateMsg : uplinkMsg.getAlarmUpdateMsgList()) { + result.add(((AlarmProcessor) ctx.getAlarmEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processAlarmMsgFromEdge(edge.getTenantId(), edge.getId(), alarmUpdateMsg)); + } + } + if (uplinkMsg.getAlarmCommentUpdateMsgCount() > 0) { + for (AlarmCommentUpdateMsg alarmCommentUpdateMsg : uplinkMsg.getAlarmCommentUpdateMsgList()) { + result.add(((AlarmProcessor) ctx.getAlarmEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processAlarmCommentMsgFromEdge(edge.getTenantId(), edge.getId(), alarmCommentUpdateMsg)); + } + } + if (uplinkMsg.getRelationUpdateMsgCount() > 0) { + for (RelationUpdateMsg relationUpdateMsg : uplinkMsg.getRelationUpdateMsgList()) { + result.add(((RelationProcessor) ctx.getRelationEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processRelationMsgFromEdge(edge.getTenantId(), edge, relationUpdateMsg)); + } + } + if (uplinkMsg.getDashboardUpdateMsgCount() > 0) { + for (DashboardUpdateMsg dashboardUpdateMsg : uplinkMsg.getDashboardUpdateMsgList()) { + result.add(((DashboardProcessor) ctx.getDashboardEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processDashboardMsgFromEdge(edge.getTenantId(), edge, dashboardUpdateMsg)); + } + } + if (uplinkMsg.getResourceUpdateMsgCount() > 0) { + for (ResourceUpdateMsg resourceUpdateMsg : uplinkMsg.getResourceUpdateMsgList()) { + result.add(((ResourceProcessor) ctx.getResourceEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processResourceMsgFromEdge(edge.getTenantId(), edge, resourceUpdateMsg)); + } + } + if (uplinkMsg.getRuleChainMetadataRequestMsgCount() > 0) { + for (RuleChainMetadataRequestMsg ruleChainMetadataRequestMsg : uplinkMsg.getRuleChainMetadataRequestMsgList()) { + result.add(ctx.getEdgeRequestsService().processRuleChainMetadataRequestMsg(edge.getTenantId(), edge, ruleChainMetadataRequestMsg)); + } + } + if (uplinkMsg.getAttributesRequestMsgCount() > 0) { + for (AttributesRequestMsg attributesRequestMsg : uplinkMsg.getAttributesRequestMsgList()) { + result.add(ctx.getEdgeRequestsService().processAttributesRequestMsg(edge.getTenantId(), edge, attributesRequestMsg)); + } + } + if (uplinkMsg.getRelationRequestMsgCount() > 0) { + for (RelationRequestMsg relationRequestMsg : uplinkMsg.getRelationRequestMsgList()) { + result.add(ctx.getEdgeRequestsService().processRelationRequestMsg(edge.getTenantId(), edge, relationRequestMsg)); + } + } + if (uplinkMsg.getUserCredentialsRequestMsgCount() > 0) { + for (UserCredentialsRequestMsg userCredentialsRequestMsg : uplinkMsg.getUserCredentialsRequestMsgList()) { + result.add(ctx.getEdgeRequestsService().processUserCredentialsRequestMsg(edge.getTenantId(), edge, userCredentialsRequestMsg)); + } + } + if (uplinkMsg.getDeviceCredentialsRequestMsgCount() > 0) { + for (DeviceCredentialsRequestMsg deviceCredentialsRequestMsg : uplinkMsg.getDeviceCredentialsRequestMsgList()) { + result.add(ctx.getEdgeRequestsService().processDeviceCredentialsRequestMsg(edge.getTenantId(), edge, deviceCredentialsRequestMsg)); + } + } + if (uplinkMsg.getDeviceRpcCallMsgCount() > 0) { + for (DeviceRpcCallMsg deviceRpcCallMsg : uplinkMsg.getDeviceRpcCallMsgList()) { + result.add(((DeviceProcessor) ctx.getDeviceEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processDeviceRpcCallFromEdge(edge.getTenantId(), edge, deviceRpcCallMsg)); + } + } + if (uplinkMsg.getWidgetBundleTypesRequestMsgCount() > 0) { + for (WidgetBundleTypesRequestMsg widgetBundleTypesRequestMsg : uplinkMsg.getWidgetBundleTypesRequestMsgList()) { + result.add(ctx.getEdgeRequestsService().processWidgetBundleTypesRequestMsg(edge.getTenantId(), edge, widgetBundleTypesRequestMsg)); + } + } + if (uplinkMsg.getEntityViewsRequestMsgCount() > 0) { + for (EntityViewsRequestMsg entityViewRequestMsg : uplinkMsg.getEntityViewsRequestMsgList()) { + result.add(ctx.getEdgeRequestsService().processEntityViewsRequestMsg(edge.getTenantId(), edge, entityViewRequestMsg)); + } + } + } catch (Exception e) { + String failureMsg = String.format("Can't process uplink msg [%s] from edge", uplinkMsg); + log.error("[{}][{}] Can't process uplink msg [{}]", edge.getTenantId(), sessionId, uplinkMsg, e); + ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(edge.getTenantId()).edgeId(edge.getId()) + .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(e.getMessage()).build()); + return Futures.immediateFailedFuture(e); + } + return Futures.allAsList(result); + } + + @Override + public void close() { + log.debug("[{}][{}] Closing session", this.tenantId, sessionId); + connected = false; + try { + outputStream.onCompleted(); + } catch (Exception e) { + log.debug("[{}][{}] Failed to close output stream: {}", this.tenantId, sessionId, e.getMessage()); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java index 5d45e1c3d5..53368dfc06 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java @@ -57,7 +57,11 @@ import org.thingsboard.server.common.msg.edge.ToEdgeSyncRequest; import org.thingsboard.server.gen.edge.v1.EdgeRpcServiceGrpc; import org.thingsboard.server.gen.edge.v1.RequestMsg; import org.thingsboard.server.gen.edge.v1.ResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeContextComponent; import org.thingsboard.server.service.state.DefaultDeviceStateService; @@ -85,7 +89,7 @@ import java.util.function.Consumer; @TbCoreComponent public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase implements EdgeRpcService { - private final ConcurrentMap sessions = new ConcurrentHashMap<>(); + private final ConcurrentMap sessions = new ConcurrentHashMap<>(); private final ConcurrentMap sessionNewEventsLocks = new ConcurrentHashMap<>(); private final Map sessionNewEvents = new HashMap<>(); private final ConcurrentMap> sessionEdgeEventChecks = new ConcurrentHashMap<>(); @@ -118,6 +122,9 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i @Value("${edges.max_high_priority_queue_size_per_session:10000}") private int maxHighPriorityQueueSizePerSession; + @Value("#{ '${queue.type:null}' == 'kafka' }") + private boolean isKafkaSupported; + @Autowired @Lazy private EdgeContextComponent ctx; @@ -134,6 +141,9 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i @Autowired private TbTransactionalCache edgeIdServiceIdCache; + @Autowired + private TbCoreQueueFactory tbCoreQueueFactory; + private Server server; private ScheduledExecutorService edgeEventProcessingExecutorService; @@ -202,7 +212,16 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i @Override public StreamObserver handleMsgs(StreamObserver outputStream) { - return new EdgeGrpcSession(ctx, + if (isKafkaSupported) { + return new KafkaEdgeGrpcSession(ctx, + outputStream, + this::onEdgeConnect, + this::onEdgeDisconnect, + sendDownlinkExecutorService, + this.maxInboundMessageSize, + this.maxHighPriorityQueueSizePerSession).getInputStream(); + } + return new PostgresEdgeGrpcSession(ctx, outputStream, this::onEdgeConnect, this::onEdgeDisconnect, @@ -239,7 +258,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i @Override public void updateEdge(TenantId tenantId, Edge edge) { - EdgeGrpcSession session = sessions.get(edge.getId()); + AbstractEdgeGrpcSession session = sessions.get(edge.getId()); if (session != null && session.isConnected()) { log.debug("[{}] Updating configuration for edge [{}] [{}]", tenantId, edge.getName(), edge.getId()); session.onConfigurationUpdate(edge); @@ -250,7 +269,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i @Override public void deleteEdge(TenantId tenantId, EdgeId edgeId) { - EdgeGrpcSession session = sessions.get(edgeId); + AbstractEdgeGrpcSession session = sessions.get(edgeId); if (session != null && session.isConnected()) { log.info("[{}] Closing and removing session for edge [{}]", tenantId, edgeId); session.close(); @@ -267,7 +286,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } private void onEdgeEventUpdate(TenantId tenantId, EdgeId edgeId) { - EdgeGrpcSession session = sessions.get(edgeId); + AbstractEdgeGrpcSession session = sessions.get(edgeId); if (session != null && session.isConnected()) { log.trace("[{}] onEdgeEventUpdate [{}]", tenantId, edgeId.getId()); updateSessionEventsFlag(tenantId, edgeId); @@ -278,7 +297,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i TenantId tenantId = msg.getTenantId(); EdgeEvent edgeEvent = msg.getEdgeEvent(); EdgeId edgeId = edgeEvent.getEdgeId(); - EdgeGrpcSession session = sessions.get(edgeId); + AbstractEdgeGrpcSession session = sessions.get(edgeId); if (session != null && session.isConnected()) { log.trace("[{}] onEdgeEvent [{}]", tenantId, edgeId); session.addEventToHighPriorityQueue(edgeEvent); @@ -299,7 +318,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } } - private void onEdgeConnect(EdgeId edgeId, EdgeGrpcSession edgeGrpcSession) { + private void onEdgeConnect(EdgeId edgeId, AbstractEdgeGrpcSession edgeGrpcSession) { Edge edge = edgeGrpcSession.getEdge(); TenantId tenantId = edge.getTenantId(); log.info("[{}][{}] edge [{}] connected successfully.", tenantId, edgeGrpcSession.getSessionId(), edgeId); @@ -315,13 +334,17 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i long lastConnectTs = System.currentTimeMillis(); save(tenantId, edgeId, DefaultDeviceStateService.LAST_CONNECT_TIME, lastConnectTs); edgeIdServiceIdCache.put(edgeId, serviceInfoProvider.getServiceId()); + if (isKafkaSupported) { + TbQueueConsumer> consumer = tbCoreQueueFactory.createEdgeEventMsgConsumer(tenantId, edgeId, serviceInfoProvider.getServiceId()); + ((KafkaEdgeGrpcSession) edgeGrpcSession).initConsumer(consumer); + } pushRuleEngineMessage(tenantId, edge, lastConnectTs, TbMsgType.CONNECT_EVENT); cancelScheduleEdgeEventsCheck(edgeId); scheduleEdgeEventsCheck(edgeGrpcSession); } private void startSyncProcess(TenantId tenantId, EdgeId edgeId, UUID requestId, String requestServiceId) { - EdgeGrpcSession session = sessions.get(edgeId); + AbstractEdgeGrpcSession session = sessions.get(edgeId); if (session != null) { if (!session.isSyncCompleted()) { clusterService.pushEdgeSyncResponseToCore(new FromEdgeSyncResponse(requestId, tenantId, edgeId, false, "Sync process is active at the moment"), requestServiceId); @@ -341,7 +364,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i ToEdgeSyncRequest request = new ToEdgeSyncRequest(UUID.randomUUID(), tenantId, edgeId, serviceInfoProvider.getServiceId()); UUID requestId = request.getId(); - EdgeGrpcSession session = sessions.get(request.getEdgeId()); + AbstractEdgeGrpcSession session = sessions.get(request.getEdgeId()); if (session != null && !session.isSyncCompleted()) { responseConsumer.accept(new FromEdgeSyncResponse(requestId, request.getTenantId(), request.getEdgeId(), false, "Sync process is active at the moment")); } else { @@ -375,7 +398,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } } - private void scheduleEdgeEventsCheck(EdgeGrpcSession session) { + private void scheduleEdgeEventsCheck(AbstractEdgeGrpcSession session) { EdgeId edgeId = session.getEdge().getId(); UUID tenantId = session.getEdge().getTenantId().getId(); if (sessions.containsKey(edgeId)) { @@ -387,7 +410,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i if (Boolean.TRUE.equals(sessionNewEvents.get(edgeId))) { log.trace("[{}][{}] Set session new events flag to false", tenantId, edgeId.getId()); sessionNewEvents.put(edgeId, false); - Futures.addCallback(session.processEdgeEvents(), new FutureCallback<>() { + Futures.addCallback(session.processEdgeEvents(), new FutureCallback() { @Override public void onSuccess(Boolean newEventsAdded) { if (Boolean.TRUE.equals(newEventsAdded)) { @@ -395,7 +418,6 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } scheduleEdgeEventsCheck(session); } - @Override public void onFailure(Throwable t) { log.warn("[{}] Failed to process edge events for edge [{}]!", tenantId, session.getEdge().getId().getId(), t); @@ -434,7 +456,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i private void onEdgeDisconnect(Edge edge, UUID sessionId) { EdgeId edgeId = edge.getId(); log.info("[{}][{}] edge disconnected!", edgeId, sessionId); - EdgeGrpcSession toRemove = sessions.get(edgeId); + AbstractEdgeGrpcSession toRemove = sessions.get(edgeId); if (toRemove.getSessionId().equals(sessionId)) { toRemove = sessions.remove(edgeId); final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock()); @@ -444,6 +466,9 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } finally { newEventLock.unlock(); } + if (isKafkaSupported) { + ((KafkaEdgeGrpcSession) toRemove).stopConsumer(); + } TenantId tenantId = toRemove.getEdge().getTenantId(); save(tenantId, edgeId, DefaultDeviceStateService.ACTIVITY_STATE, false); long lastDisconnectTs = System.currentTimeMillis(); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index d0d10f0174..6a72b40b80 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -15,943 +15,14 @@ */ package org.thingsboard.server.service.edge.rpc; -import com.datastax.oss.driver.api.core.uuid.Uuids; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.SettableFuture; -import io.grpc.stub.StreamObserver; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.checkerframework.checker.nullness.qual.Nullable; -import org.springframework.data.util.Pair; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.edge.EdgeEvent; -import org.thingsboard.server.common.data.id.EdgeId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.limit.LimitedApi; -import org.thingsboard.server.common.data.notification.rule.trigger.EdgeCommunicationFailureTrigger; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.page.SortOrder; -import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; -import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; -import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; -import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; -import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; -import org.thingsboard.server.gen.edge.v1.AttributesRequestMsg; -import org.thingsboard.server.gen.edge.v1.ConnectRequestMsg; -import org.thingsboard.server.gen.edge.v1.ConnectResponseCode; -import org.thingsboard.server.gen.edge.v1.ConnectResponseMsg; -import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DeviceCredentialsRequestMsg; -import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DeviceRpcCallMsg; -import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.DownlinkResponseMsg; -import org.thingsboard.server.gen.edge.v1.EdgeConfiguration; -import org.thingsboard.server.gen.edge.v1.EdgeUpdateMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; -import org.thingsboard.server.gen.edge.v1.EntityDataProto; -import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; -import org.thingsboard.server.gen.edge.v1.EntityViewsRequestMsg; -import org.thingsboard.server.gen.edge.v1.RelationRequestMsg; -import org.thingsboard.server.gen.edge.v1.RelationUpdateMsg; -import org.thingsboard.server.gen.edge.v1.RequestMsg; -import org.thingsboard.server.gen.edge.v1.RequestMsgType; -import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; -import org.thingsboard.server.gen.edge.v1.ResponseMsg; -import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg; -import org.thingsboard.server.gen.edge.v1.SyncCompletedMsg; -import org.thingsboard.server.gen.edge.v1.UplinkMsg; -import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; -import org.thingsboard.server.gen.edge.v1.UserCredentialsRequestMsg; -import org.thingsboard.server.gen.edge.v1.WidgetBundleTypesRequestMsg; -import org.thingsboard.server.service.edge.EdgeContextComponent; -import org.thingsboard.server.service.edge.rpc.fetch.EdgeEventFetcher; -import org.thingsboard.server.service.edge.rpc.fetch.GeneralEdgeEventFetcher; -import org.thingsboard.server.service.edge.rpc.processor.alarm.AlarmProcessor; -import org.thingsboard.server.service.edge.rpc.processor.asset.AssetProcessor; -import org.thingsboard.server.service.edge.rpc.processor.asset.profile.AssetProfileProcessor; -import org.thingsboard.server.service.edge.rpc.processor.dashboard.DashboardProcessor; -import org.thingsboard.server.service.edge.rpc.processor.device.DeviceProcessor; -import org.thingsboard.server.service.edge.rpc.processor.device.profile.DeviceProfileProcessor; -import org.thingsboard.server.service.edge.rpc.processor.entityview.EntityViewProcessor; -import org.thingsboard.server.service.edge.rpc.processor.relation.RelationProcessor; -import org.thingsboard.server.service.edge.rpc.processor.resource.ResourceProcessor; -import java.io.Closeable; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.BiConsumer; +public interface EdgeGrpcSession { -@Slf4j -@Data -public final class EdgeGrpcSession implements Closeable { + void onConfigurationUpdate(Edge edge); - private static final ReentrantLock downlinkMsgLock = new ReentrantLock(); - private static final ConcurrentLinkedQueue highPriorityQueue = new ConcurrentLinkedQueue<>(); + void startSyncProcess(boolean fullSync); - private static final int MAX_DOWNLINK_ATTEMPTS = 10; // max number of attempts to send downlink message if edge connected - - private static final String QUEUE_START_TS_ATTR_KEY = "queueStartTs"; - private static final String QUEUE_START_SEQ_ID_ATTR_KEY = "queueStartSeqId"; - private static final String RATE_LIMIT_REACHED = "Rate limit reached"; - - private final UUID sessionId; - private final BiConsumer sessionOpenListener; - private final BiConsumer sessionCloseListener; - - private final EdgeSessionState sessionState = new EdgeSessionState(); - - private EdgeContextComponent ctx; - private Edge edge; - private TenantId tenantId; - private StreamObserver inputStream; - private StreamObserver outputStream; - private boolean connected; - private volatile boolean syncCompleted; - - private Long newStartTs; - private Long previousStartTs; - private Long newStartSeqId; - private Long previousStartSeqId; - private Long seqIdEnd; - - private EdgeVersion edgeVersion; - - private int maxInboundMessageSize; - private int clientMaxInboundMessageSize; - private int maxHighPriorityQueueSizePerSession; - - private ScheduledExecutorService sendDownlinkExecutorService; - - EdgeGrpcSession(EdgeContextComponent ctx, StreamObserver outputStream, BiConsumer sessionOpenListener, - BiConsumer sessionCloseListener, ScheduledExecutorService sendDownlinkExecutorService, int maxInboundMessageSize, int maxHighPriorityQueueSizePerSession) { - this.sessionId = UUID.randomUUID(); - this.ctx = ctx; - this.outputStream = outputStream; - this.sessionOpenListener = sessionOpenListener; - this.sessionCloseListener = sessionCloseListener; - this.sendDownlinkExecutorService = sendDownlinkExecutorService; - this.maxInboundMessageSize = maxInboundMessageSize; - this.maxHighPriorityQueueSizePerSession = maxHighPriorityQueueSizePerSession; - initInputStream(); - } - - private void initInputStream() { - this.inputStream = new StreamObserver<>() { - @Override - public void onNext(RequestMsg requestMsg) { - if (!connected && requestMsg.getMsgType().equals(RequestMsgType.CONNECT_RPC_MESSAGE)) { - ConnectResponseMsg responseMsg = processConnect(requestMsg.getConnectRequestMsg()); - outputStream.onNext(ResponseMsg.newBuilder() - .setConnectResponseMsg(responseMsg) - .build()); - if (ConnectResponseCode.ACCEPTED != responseMsg.getResponseCode()) { - outputStream.onError(new RuntimeException(responseMsg.getErrorMsg())); - } else { - if (requestMsg.getConnectRequestMsg().hasMaxInboundMessageSize()) { - log.debug("[{}][{}] Client max inbound message size: {}", tenantId, sessionId, requestMsg.getConnectRequestMsg().getMaxInboundMessageSize()); - clientMaxInboundMessageSize = requestMsg.getConnectRequestMsg().getMaxInboundMessageSize(); - } - connected = true; - } - } - if (connected) { - if (requestMsg.getMsgType().equals(RequestMsgType.SYNC_REQUEST_RPC_MESSAGE)) { - if (requestMsg.hasSyncRequestMsg()) { - boolean fullSync = false; - if (requestMsg.getSyncRequestMsg().hasFullSync()) { - fullSync = requestMsg.getSyncRequestMsg().getFullSync(); - } - startSyncProcess(fullSync); - } else { - syncCompleted = true; - } - } - if (requestMsg.getMsgType().equals(RequestMsgType.UPLINK_RPC_MESSAGE)) { - if (requestMsg.hasUplinkMsg()) { - onUplinkMsg(requestMsg.getUplinkMsg()); - } - if (requestMsg.hasDownlinkResponseMsg()) { - onDownlinkResponse(requestMsg.getDownlinkResponseMsg()); - } - } - } - } - - @Override - public void onError(Throwable t) { - log.error("[{}][{}] Stream was terminated due to error:", tenantId, sessionId, t); - closeSession(); - } - - @Override - public void onCompleted() { - log.info("[{}][{}] Stream was closed and completed successfully!", tenantId, sessionId); - closeSession(); - } - - private void closeSession() { - connected = false; - if (edge != null) { - try { - sessionCloseListener.accept(edge, sessionId); - } catch (Exception ignored) { - } - } - try { - outputStream.onCompleted(); - } catch (Exception ignored) { - } - } - }; - } - - public void startSyncProcess(boolean fullSync) { - log.info("[{}][{}][{}] Staring edge sync process", this.tenantId, edge.getId(), this.sessionId); - syncCompleted = false; - interruptGeneralProcessingOnSync(); - doSync(new EdgeSyncCursor(ctx, edge, fullSync)); - } - - private void doSync(EdgeSyncCursor cursor) { - if (cursor.hasNext()) { - EdgeEventFetcher next = cursor.getNext(); - log.info("[{}][{}] starting sync process, cursor current idx = {}, class = {}", - this.tenantId, edge.getId(), cursor.getCurrentIdx(), next.getClass().getSimpleName()); - ListenableFuture> future = startProcessingEdgeEvents(next); - Futures.addCallback(future, new FutureCallback<>() { - @Override - public void onSuccess(@Nullable Pair result) { - doSync(cursor); - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}][{}] Exception during sync process", tenantId, edge.getId(), t); - } - }, ctx.getGrpcCallbackExecutorService()); - } else { - log.info("[{}][{}] sync process completed", this.tenantId, edge.getId()); - DownlinkMsg syncCompleteDownlinkMsg = DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .setSyncCompletedMsg(SyncCompletedMsg.newBuilder().build()) - .build(); - Futures.addCallback(sendDownlinkMsgsPack(Collections.singletonList(syncCompleteDownlinkMsg)), new FutureCallback<>() { - @Override - public void onSuccess(Boolean isInterrupted) { - markSyncCompletedSendEdgeEventUpdate(); - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}][{}] Exception during sending sync complete", tenantId, edge.getId(), t); - markSyncCompletedSendEdgeEventUpdate(); - } - }, ctx.getGrpcCallbackExecutorService()); - } - } - - private void markSyncCompletedSendEdgeEventUpdate() { - syncCompleted = true; - ctx.getClusterService().onEdgeEventUpdate(new EdgeEventUpdateMsg(edge.getTenantId(), edge.getId())); - } - - private void onUplinkMsg(UplinkMsg uplinkMsg) { - if (isRateLimitViolated(uplinkMsg)) { - return; - } - ListenableFuture> future = processUplinkMsg(uplinkMsg); - Futures.addCallback(future, new FutureCallback<>() { - @Override - public void onSuccess(@Nullable List result) { - sendResponseMessage(uplinkMsg.getUplinkMsgId(), true, null); - } - - @Override - public void onFailure(Throwable t) { - String errorMsg = EdgeUtils.createErrorMsgFromRootCauseAndStackTrace(t); - sendResponseMessage(uplinkMsg.getUplinkMsgId(), false, errorMsg); - } - }, ctx.getGrpcCallbackExecutorService()); - } - - private boolean isRateLimitViolated(UplinkMsg uplinkMsg) { - if (!ctx.getRateLimitService().checkRateLimit(LimitedApi.EDGE_UPLINK_MESSAGES, tenantId) || - !ctx.getRateLimitService().checkRateLimit(LimitedApi.EDGE_UPLINK_MESSAGES_PER_EDGE, tenantId, edge.getId())) { - String errorMsg = String.format("Failed to process uplink message. %s", RATE_LIMIT_REACHED); - sendResponseMessage(uplinkMsg.getUplinkMsgId(), false, errorMsg); - return true; - } - return false; - } - - private void sendResponseMessage(int uplinkMsgId, boolean success, String errorMsg) { - UplinkResponseMsg.Builder responseBuilder = UplinkResponseMsg.newBuilder() - .setUplinkMsgId(uplinkMsgId) - .setSuccess(success); - if (errorMsg != null) { - responseBuilder.setErrorMsg(errorMsg); - } - sendDownlinkMsg(ResponseMsg.newBuilder() - .setUplinkResponseMsg(responseBuilder.build()) - .build()); - } - - private void onDownlinkResponse(DownlinkResponseMsg msg) { - try { - if (msg.getSuccess()) { - sessionState.getPendingMsgsMap().remove(msg.getDownlinkMsgId()); - log.debug("[{}][{}] Msg has been processed successfully! Msg Id: [{}], Msg: {}", this.tenantId, edge.getRoutingKey(), msg.getDownlinkMsgId(), msg); - } else { - log.error("[{}][{}] Msg processing failed! Msg Id: [{}], Error msg: {}", this.tenantId, edge.getRoutingKey(), msg.getDownlinkMsgId(), msg.getErrorMsg()); - } - if (sessionState.getPendingMsgsMap().isEmpty()) { - log.debug("[{}][{}] Pending msgs map is empty. Stopping current iteration", this.tenantId, edge.getRoutingKey()); - stopCurrentSendDownlinkMsgsTask(false); - } - } catch (Exception e) { - log.error("[{}][{}] Can't process downlink response message [{}]", this.tenantId, this.sessionId, msg, e); - } - } - - private void sendDownlinkMsg(ResponseMsg downlinkMsg) { - if (downlinkMsg.getDownlinkMsg().getWidgetTypeUpdateMsgCount() > 0) { - log.trace("[{}][{}] Sending downlink widgetTypeUpdateMsg, downlinkMsgId = {}", this.tenantId, this.sessionId, downlinkMsg.getDownlinkMsg().getDownlinkMsgId()); - } else { - log.trace("[{}][{}] Sending downlink msg [{}]", this.tenantId, this.sessionId, downlinkMsg); - } - if (isConnected()) { - downlinkMsgLock.lock(); - try { - outputStream.onNext(downlinkMsg); - } catch (Exception e) { - log.error("[{}][{}] Failed to send downlink message [{}]", this.tenantId, this.sessionId, downlinkMsg, e); - connected = false; - sessionCloseListener.accept(edge, sessionId); - } finally { - downlinkMsgLock.unlock(); - } - log.trace("[{}][{}] Response msg successfully sent. downlinkMsgId = {}", this.tenantId, this.sessionId, downlinkMsg.getDownlinkMsg().getDownlinkMsgId()); - } - } - - void onConfigurationUpdate(Edge edge) { - log.debug("[{}] onConfigurationUpdate [{}]", this.sessionId, edge); - this.edge = edge; - this.tenantId = edge.getTenantId(); - EdgeUpdateMsg edgeConfig = EdgeUpdateMsg.newBuilder() - .setConfiguration(ctx.getEdgeMsgConstructor().constructEdgeConfiguration(edge)).build(); - ResponseMsg edgeConfigMsg = ResponseMsg.newBuilder() - .setEdgeUpdateMsg(edgeConfig) - .build(); - sendDownlinkMsg(edgeConfigMsg); - } - - ListenableFuture processEdgeEvents() throws Exception { - SettableFuture result = SettableFuture.create(); - log.trace("[{}][{}] starting processing edge events", this.tenantId, this.sessionId); - if (isConnected() && isSyncCompleted()) { - Pair startTsAndSeqId = getQueueStartTsAndSeqId().get(); - this.previousStartTs = startTsAndSeqId.getFirst(); - this.previousStartSeqId = startTsAndSeqId.getSecond(); - GeneralEdgeEventFetcher fetcher = new GeneralEdgeEventFetcher( - this.previousStartTs, - this.previousStartSeqId, - this.seqIdEnd, - false, - Integer.toUnsignedLong(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount()), - ctx.getEdgeEventService()); - Futures.addCallback(startProcessingEdgeEvents(fetcher), new FutureCallback<>() { - @Override - public void onSuccess(@Nullable Pair newStartTsAndSeqId) { - if (newStartTsAndSeqId != null) { - ListenableFuture> updateFuture = updateQueueStartTsAndSeqId(newStartTsAndSeqId); - Futures.addCallback(updateFuture, new FutureCallback<>() { - @Override - public void onSuccess(@Nullable List list) { - log.debug("[{}][{}] queue offset was updated [{}]", tenantId, sessionId, newStartTsAndSeqId); - if (fetcher.isSeqIdNewCycleStarted()) { - seqIdEnd = fetcher.getSeqIdEnd(); - boolean newEventsAvailable = isNewEdgeEventsAvailable(); - result.set(newEventsAvailable); - } else { - seqIdEnd = null; - boolean newEventsAvailable = isSeqIdStartedNewCycle(); - if (!newEventsAvailable) { - newEventsAvailable = isNewEdgeEventsAvailable(); - } - result.set(newEventsAvailable); - } - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}][{}] Failed to update queue offset [{}]", tenantId, sessionId, newStartTsAndSeqId, t); - result.setException(t); - } - }, ctx.getGrpcCallbackExecutorService()); - } else { - log.trace("[{}][{}] newStartTsAndSeqId is null. Skipping iteration without db update", tenantId, sessionId); - result.set(null); - } - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}][{}] Failed to process events", tenantId, sessionId, t); - result.setException(t); - } - }, ctx.getGrpcCallbackExecutorService()); - } else { - log.trace("[{}][{}] edge is not connected or sync is not completed. Skipping iteration", tenantId, sessionId); - result.set(null); - } - return result; - } - - private ListenableFuture> startProcessingEdgeEvents(EdgeEventFetcher fetcher) { - SettableFuture> result = SettableFuture.create(); - PageLink pageLink = fetcher.getPageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount()); - processEdgeEvents(fetcher, pageLink, result); - return result; - } - - private void processEdgeEvents(EdgeEventFetcher fetcher, PageLink pageLink, SettableFuture> result) { - try { - if (!highPriorityQueue.isEmpty()) { - processHighPriorityEvents(); - } - PageData pageData = fetcher.fetchEdgeEvents(edge.getTenantId(), edge, pageLink); - if (isConnected() && !pageData.getData().isEmpty()) { - log.trace("[{}][{}][{}] event(s) are going to be processed.", this.tenantId, this.sessionId, pageData.getData().size()); - List downlinkMsgsPack = convertToDownlinkMsgsPack(pageData.getData()); - Futures.addCallback(sendDownlinkMsgsPack(downlinkMsgsPack), new FutureCallback<>() { - @Override - public void onSuccess(@Nullable Boolean isInterrupted) { - if (Boolean.TRUE.equals(isInterrupted)) { - log.debug("[{}][{}][{}] Send downlink messages task was interrupted", tenantId, edge.getId(), sessionId); - result.set(null); - } else { - if (isConnected() && pageData.hasNext()) { - processEdgeEvents(fetcher, pageLink.nextPageLink(), result); - } else { - EdgeEvent latestEdgeEvent = pageData.getData().get(pageData.getData().size() - 1); - UUID idOffset = latestEdgeEvent.getUuidId(); - if (idOffset != null) { - Long newStartTs = Uuids.unixTimestamp(idOffset); - long newStartSeqId = latestEdgeEvent.getSeqId(); - result.set(Pair.of(newStartTs, newStartSeqId)); - } else { - result.set(null); - } - } - } - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}] Failed to send downlink msgs pack", sessionId, t); - result.setException(t); - } - }, ctx.getGrpcCallbackExecutorService()); - } else { - log.trace("[{}] no event(s) found. Stop processing edge events", this.sessionId); - result.set(null); - } - } catch (Exception e) { - log.error("[{}] Failed to fetch edge events", this.sessionId, e); - result.setException(e); - } - } - - private void processHighPriorityEvents() { - try { - List highPriorityEvents = new ArrayList<>(); - EdgeEvent event; - while ((event = highPriorityQueue.poll()) != null) { - highPriorityEvents.add(event); - } - List downlinkMsgsPack = convertToDownlinkMsgsPack(highPriorityEvents); - sendDownlinkMsgsPack(downlinkMsgsPack).get(); - } catch (Exception e) { - log.error("[{}] Failed to process high priority events", this.sessionId, e); - } - } - - private ListenableFuture sendDownlinkMsgsPack(List downlinkMsgsPack) { - interruptPreviousSendDownlinkMsgsTask(); - - sessionState.setSendDownlinkMsgsFuture(SettableFuture.create()); - sessionState.getPendingMsgsMap().clear(); - - downlinkMsgsPack.forEach(msg -> sessionState.getPendingMsgsMap().put(msg.getDownlinkMsgId(), msg)); - scheduleDownlinkMsgsPackSend(1); - - return sessionState.getSendDownlinkMsgsFuture(); - } - - private void scheduleDownlinkMsgsPackSend(int attempt) { - Runnable sendDownlinkMsgsTask = () -> { - try { - if (isConnected() && sessionState.getPendingMsgsMap().values().size() > 0) { - List copy = new ArrayList<>(sessionState.getPendingMsgsMap().values()); - if (attempt > 1) { - String error = "Failed to deliver the batch"; - String failureMsg = String.format("{%s}: {%s}", error, copy); - if (attempt == 2) { - // Send a failure notification only on the second attempt. - // This ensures that failure alerts are sent just once to avoid redundant notifications. - ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId) - .edgeId(edge.getId()).customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(error).build()); - } - log.warn("[{}][{}] {}, attempt: {}", this.tenantId, this.sessionId, failureMsg, attempt); - } - log.trace("[{}][{}][{}] downlink msg(s) are going to be send.", this.tenantId, this.sessionId, copy.size()); - for (DownlinkMsg downlinkMsg : copy) { - if (this.clientMaxInboundMessageSize != 0 && downlinkMsg.getSerializedSize() > this.clientMaxInboundMessageSize) { - String error = String.format("Client max inbound message size %s is exceeded. Please increase value of CLOUD_RPC_MAX_INBOUND_MESSAGE_SIZE " + - "env variable on the edge and restart it.", this.clientMaxInboundMessageSize); - String message = String.format("Downlink msg size %s exceeds client max inbound message size %s. " + - "Please increase value of CLOUD_RPC_MAX_INBOUND_MESSAGE_SIZE env variable on the edge and restart it.", downlinkMsg.getSerializedSize(), this.clientMaxInboundMessageSize); - log.error("[{}][{}][{}] {} Message {}", this.tenantId, edge.getId(), this.sessionId, message, downlinkMsg); - ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId) - .edgeId(edge.getId()).customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(message).error(error).build()); - sessionState.getPendingMsgsMap().remove(downlinkMsg.getDownlinkMsgId()); - } else { - sendDownlinkMsg(ResponseMsg.newBuilder() - .setDownlinkMsg(downlinkMsg) - .build()); - } - } - if (attempt < MAX_DOWNLINK_ATTEMPTS) { - scheduleDownlinkMsgsPackSend(attempt + 1); - } else { - String failureMsg = String.format("Failed to deliver messages: %s", copy); - log.warn("[{}][{}] Failed to deliver the batch after {} attempts. Next messages are going to be discarded {}", - this.tenantId, this.sessionId, MAX_DOWNLINK_ATTEMPTS, copy); - ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) - .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg) - .error("Failed to deliver messages after " + MAX_DOWNLINK_ATTEMPTS + " attempts").build()); - stopCurrentSendDownlinkMsgsTask(false); - } - } else { - stopCurrentSendDownlinkMsgsTask(false); - } - } catch (Exception e) { - log.warn("[{}][{}] Failed to send downlink msgs. Error msg {}", this.tenantId, this.sessionId, e.getMessage(), e); - stopCurrentSendDownlinkMsgsTask(true); - } - }; - - if (attempt == 1) { - sendDownlinkExecutorService.submit(sendDownlinkMsgsTask); - } else { - sessionState.setScheduledSendDownlinkTask( - sendDownlinkExecutorService.schedule( - sendDownlinkMsgsTask, - ctx.getEdgeEventStorageSettings().getSleepIntervalBetweenBatches(), - TimeUnit.MILLISECONDS) - ); - } - } - - private List convertToDownlinkMsgsPack(List edgeEvents) { - List result = new ArrayList<>(); - for (EdgeEvent edgeEvent : edgeEvents) { - log.trace("[{}][{}] converting edge event to downlink msg [{}]", this.tenantId, this.sessionId, edgeEvent); - DownlinkMsg downlinkMsg = null; - try { - switch (edgeEvent.getAction()) { - case UPDATED, ADDED, DELETED, ASSIGNED_TO_EDGE, UNASSIGNED_FROM_EDGE, ALARM_ACK, ALARM_CLEAR, ALARM_DELETE, - CREDENTIALS_UPDATED, RELATION_ADD_OR_UPDATE, RELATION_DELETED, RPC_CALL, - ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER, ADDED_COMMENT, UPDATED_COMMENT, DELETED_COMMENT -> { - downlinkMsg = convertEntityEventToDownlink(edgeEvent); - if (downlinkMsg != null && downlinkMsg.getWidgetTypeUpdateMsgCount() > 0) { - log.trace("[{}][{}] widgetTypeUpdateMsg message processed, downlinkMsgId = {}", this.tenantId, this.sessionId, downlinkMsg.getDownlinkMsgId()); - } else { - log.trace("[{}][{}] entity message processed [{}]", this.tenantId, this.sessionId, downlinkMsg); - } - } - case ATTRIBUTES_UPDATED, POST_ATTRIBUTES, ATTRIBUTES_DELETED, TIMESERIES_UPDATED -> - downlinkMsg = ctx.getTelemetryProcessor().convertTelemetryEventToDownlink(edge, edgeEvent); - default -> log.warn("[{}][{}] Unsupported action type [{}]", this.tenantId, this.sessionId, edgeEvent.getAction()); - } - } catch (Exception e) { - log.error("[{}][{}] Exception during converting edge event to downlink msg", this.tenantId, this.sessionId, e); - } - if (downlinkMsg != null) { - result.add(downlinkMsg); - } - } - return result; - } - - private ListenableFuture> getQueueStartTsAndSeqId() { - ListenableFuture> future = - ctx.getAttributesService().find(edge.getTenantId(), edge.getId(), AttributeScope.SERVER_SCOPE, Arrays.asList(QUEUE_START_TS_ATTR_KEY, QUEUE_START_SEQ_ID_ATTR_KEY)); - return Futures.transform(future, attributeKvEntries -> { - long startTs = 0L; - long startSeqId = 0L; - for (AttributeKvEntry attributeKvEntry : attributeKvEntries) { - if (QUEUE_START_TS_ATTR_KEY.equals(attributeKvEntry.getKey())) { - startTs = attributeKvEntry.getLongValue().isPresent() ? attributeKvEntry.getLongValue().get() : 0L; - } - if (QUEUE_START_SEQ_ID_ATTR_KEY.equals(attributeKvEntry.getKey())) { - startSeqId = attributeKvEntry.getLongValue().isPresent() ? attributeKvEntry.getLongValue().get() : 0L; - } - } - if (startSeqId == 0L) { - startSeqId = findStartSeqIdFromOldestEventIfAny(); - } - return Pair.of(startTs, startSeqId); - }, ctx.getGrpcCallbackExecutorService()); - } - - private boolean isSeqIdStartedNewCycle() { - try { - TimePageLink pageLink = new TimePageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount(), 0, null, null, this.newStartTs, System.currentTimeMillis()); - PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), 0L, this.previousStartSeqId == 0 ? null : this.previousStartSeqId - 1, pageLink); - return !edgeEvents.getData().isEmpty(); - } catch (Exception e) { - log.error("[{}][{}][{}] Failed to execute isSeqIdStartedNewCycle", this.tenantId, edge.getId(), sessionId, e); - } - return false; - } - - private boolean isNewEdgeEventsAvailable() { - try { - TimePageLink pageLink = new TimePageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount(), 0, null, null, this.newStartTs, System.currentTimeMillis()); - PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), this.newStartSeqId, null, pageLink); - return !edgeEvents.getData().isEmpty(); - } catch (Exception e) { - log.error("[{}][{}][{}] Failed to execute isNewEdgeEventsAvailable", this.tenantId, edge.getId(), sessionId, e); - } - return false; - } - - private long findStartSeqIdFromOldestEventIfAny() { - long startSeqId = 0L; - try { - TimePageLink pageLink = new TimePageLink(1, 0, null, new SortOrder("createdTime"), null, null); - PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), null, null, pageLink); - if (!edgeEvents.getData().isEmpty()) { - startSeqId = edgeEvents.getData().get(0).getSeqId() - 1; - } - } catch (Exception e) { - log.error("[{}][{}][{}] Failed to execute findStartSeqIdFromOldestEventIfAny", this.tenantId, edge.getId(), sessionId, e); - } - return startSeqId; - } - - private ListenableFuture> updateQueueStartTsAndSeqId(Pair pair) { - this.newStartTs = pair.getFirst(); - this.newStartSeqId = pair.getSecond(); - log.trace("[{}] updateQueueStartTsAndSeqId [{}][{}][{}]", this.sessionId, edge.getId(), this.newStartTs, this.newStartSeqId); - List attributes = Arrays.asList( - new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_TS_ATTR_KEY, this.newStartTs), System.currentTimeMillis()), - new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_SEQ_ID_ATTR_KEY, this.newStartSeqId), System.currentTimeMillis())); - return ctx.getAttributesService().save(edge.getTenantId(), edge.getId(), AttributeScope.SERVER_SCOPE, attributes); - } - - private DownlinkMsg convertEntityEventToDownlink(EdgeEvent edgeEvent) { - log.trace("[{}] Executing convertEntityEventToDownlink, edgeEvent [{}], action [{}]", this.tenantId, edgeEvent, edgeEvent.getAction()); - switch (edgeEvent.getType()) { - case EDGE: - return ctx.getEdgeProcessor().convertEdgeEventToDownlink(edgeEvent); - case DEVICE: - return ctx.getDeviceProcessor().convertDeviceEventToDownlink(edgeEvent, this.edge.getId(), this.edgeVersion); - case DEVICE_PROFILE: - return ctx.getDeviceProfileProcessor().convertDeviceProfileEventToDownlink(edgeEvent, this.edge.getId(), this.edgeVersion); - case ASSET_PROFILE: - return ctx.getAssetProfileProcessor().convertAssetProfileEventToDownlink(edgeEvent, this.edge.getId(), this.edgeVersion); - case ASSET: - return ctx.getAssetProcessor().convertAssetEventToDownlink(edgeEvent, this.edge.getId(), this.edgeVersion); - case ENTITY_VIEW: - return ctx.getEntityViewProcessor().convertEntityViewEventToDownlink(edgeEvent, this.edgeVersion); - case DASHBOARD: - return ctx.getDashboardProcessor().convertDashboardEventToDownlink(edgeEvent, this.edgeVersion); - case CUSTOMER: - return ctx.getCustomerProcessor().convertCustomerEventToDownlink(edgeEvent, this.edgeVersion); - case RULE_CHAIN: - return ctx.getRuleChainProcessor().convertRuleChainEventToDownlink(edgeEvent, this.edgeVersion); - case RULE_CHAIN_METADATA: - return ctx.getRuleChainProcessor().convertRuleChainMetadataEventToDownlink(edgeEvent, this.edgeVersion); - case ALARM: - return ctx.getAlarmProcessor().convertAlarmEventToDownlink(edgeEvent, this.edgeVersion); - case ALARM_COMMENT: - return ctx.getAlarmProcessor().convertAlarmCommentEventToDownlink(edgeEvent, this.edgeVersion); - case USER: - return ctx.getUserProcessor().convertUserEventToDownlink(edgeEvent, this.edgeVersion); - case RELATION: - return ctx.getRelationProcessor().convertRelationEventToDownlink(edgeEvent, this.edgeVersion); - case WIDGETS_BUNDLE: - return ctx.getWidgetBundleProcessor().convertWidgetsBundleEventToDownlink(edgeEvent, this.edgeVersion); - case WIDGET_TYPE: - return ctx.getWidgetTypeProcessor().convertWidgetTypeEventToDownlink(edgeEvent, this.edgeVersion); - case ADMIN_SETTINGS: - return ctx.getAdminSettingsProcessor().convertAdminSettingsEventToDownlink(edgeEvent, this.edgeVersion); - case OTA_PACKAGE: - return ctx.getOtaPackageProcessor().convertOtaPackageEventToDownlink(edgeEvent, this.edgeVersion); - case TB_RESOURCE: - return ctx.getResourceProcessor().convertResourceEventToDownlink(edgeEvent, this.edgeVersion); - case QUEUE: - return ctx.getQueueProcessor().convertQueueEventToDownlink(edgeEvent, this.edgeVersion); - case TENANT: - return ctx.getTenantProcessor().convertTenantEventToDownlink(edgeEvent, this.edgeVersion); - case TENANT_PROFILE: - return ctx.getTenantProfileProcessor().convertTenantProfileEventToDownlink(edgeEvent, this.edgeVersion); - case NOTIFICATION_RULE: - return ctx.getNotificationEdgeProcessor().convertNotificationRuleToDownlink(edgeEvent); - case NOTIFICATION_TARGET: - return ctx.getNotificationEdgeProcessor().convertNotificationTargetToDownlink(edgeEvent); - case NOTIFICATION_TEMPLATE: - return ctx.getNotificationEdgeProcessor().convertNotificationTemplateToDownlink(edgeEvent); - case OAUTH2_CLIENT: - return ctx.getOAuth2EdgeProcessor().convertOAuth2ClientEventToDownlink(edgeEvent, this.edgeVersion); - case DOMAIN: - return ctx.getOAuth2EdgeProcessor().convertOAuth2DomainEventToDownlink(edgeEvent, this.edgeVersion); - default: - log.warn("[{}] Unsupported edge event type [{}]", this.tenantId, edgeEvent); - return null; - } - } - - private ListenableFuture> processUplinkMsg(UplinkMsg uplinkMsg) { - List> result = new ArrayList<>(); - try { - if (uplinkMsg.getDeviceProfileUpdateMsgCount() > 0) { - for (DeviceProfileUpdateMsg deviceProfileUpdateMsg : uplinkMsg.getDeviceProfileUpdateMsgList()) { - result.add(((DeviceProfileProcessor) ctx.getDeviceProfileEdgeProcessorFactory().getProcessorByEdgeVersion(this.edgeVersion)) - .processDeviceProfileMsgFromEdge(edge.getTenantId(), edge, deviceProfileUpdateMsg)); - } - } - if (uplinkMsg.getDeviceUpdateMsgCount() > 0) { - for (DeviceUpdateMsg deviceUpdateMsg : uplinkMsg.getDeviceUpdateMsgList()) { - result.add(((DeviceProcessor) ctx.getDeviceEdgeProcessorFactory().getProcessorByEdgeVersion(this.edgeVersion)) - .processDeviceMsgFromEdge(edge.getTenantId(), edge, deviceUpdateMsg)); - } - } - if (uplinkMsg.getDeviceCredentialsUpdateMsgCount() > 0) { - for (DeviceCredentialsUpdateMsg deviceCredentialsUpdateMsg : uplinkMsg.getDeviceCredentialsUpdateMsgList()) { - result.add(((DeviceProcessor) ctx.getDeviceEdgeProcessorFactory().getProcessorByEdgeVersion(this.edgeVersion)) - .processDeviceCredentialsMsgFromEdge(edge.getTenantId(), edge.getId(), deviceCredentialsUpdateMsg)); - } - } - if (uplinkMsg.getAssetProfileUpdateMsgCount() > 0) { - for (AssetProfileUpdateMsg assetProfileUpdateMsg : uplinkMsg.getAssetProfileUpdateMsgList()) { - result.add(((AssetProfileProcessor) ctx.getAssetProfileEdgeProcessorFactory().getProcessorByEdgeVersion(this.edgeVersion)) - .processAssetProfileMsgFromEdge(edge.getTenantId(), edge, assetProfileUpdateMsg)); - } - } - if (uplinkMsg.getAssetUpdateMsgCount() > 0) { - for (AssetUpdateMsg assetUpdateMsg : uplinkMsg.getAssetUpdateMsgList()) { - result.add(((AssetProcessor) ctx.getAssetEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processAssetMsgFromEdge(edge.getTenantId(), edge, assetUpdateMsg)); - } - } - if (uplinkMsg.getEntityViewUpdateMsgCount() > 0) { - for (EntityViewUpdateMsg entityViewUpdateMsg : uplinkMsg.getEntityViewUpdateMsgList()) { - result.add(((EntityViewProcessor) ctx.getEntityViewProcessorFactory().getProcessorByEdgeVersion(this.edgeVersion)) - .processEntityViewMsgFromEdge(edge.getTenantId(), edge, entityViewUpdateMsg)); - } - } - if (uplinkMsg.getEntityDataCount() > 0) { - for (EntityDataProto entityData : uplinkMsg.getEntityDataList()) { - result.addAll(ctx.getTelemetryProcessor().processTelemetryMsg(edge.getTenantId(), entityData)); - } - } - if (uplinkMsg.getAlarmUpdateMsgCount() > 0) { - for (AlarmUpdateMsg alarmUpdateMsg : uplinkMsg.getAlarmUpdateMsgList()) { - result.add(((AlarmProcessor) ctx.getAlarmEdgeProcessorFactory().getProcessorByEdgeVersion(this.edgeVersion)) - .processAlarmMsgFromEdge(edge.getTenantId(), edge.getId(), alarmUpdateMsg)); - } - } - if (uplinkMsg.getAlarmCommentUpdateMsgCount() > 0) { - for (AlarmCommentUpdateMsg alarmCommentUpdateMsg : uplinkMsg.getAlarmCommentUpdateMsgList()) { - result.add(((AlarmProcessor) ctx.getAlarmEdgeProcessorFactory().getProcessorByEdgeVersion(this.edgeVersion)) - .processAlarmCommentMsgFromEdge(edge.getTenantId(), edge.getId(), alarmCommentUpdateMsg)); - } - } - if (uplinkMsg.getRelationUpdateMsgCount() > 0) { - for (RelationUpdateMsg relationUpdateMsg : uplinkMsg.getRelationUpdateMsgList()) { - result.add(((RelationProcessor) ctx.getRelationEdgeProcessorFactory().getProcessorByEdgeVersion(this.edgeVersion)) - .processRelationMsgFromEdge(edge.getTenantId(), edge, relationUpdateMsg)); - } - } - if (uplinkMsg.getDashboardUpdateMsgCount() > 0) { - for (DashboardUpdateMsg dashboardUpdateMsg : uplinkMsg.getDashboardUpdateMsgList()) { - result.add(((DashboardProcessor) ctx.getDashboardEdgeProcessorFactory().getProcessorByEdgeVersion(this.edgeVersion)) - .processDashboardMsgFromEdge(edge.getTenantId(), edge, dashboardUpdateMsg)); - } - } - if (uplinkMsg.getResourceUpdateMsgCount() > 0) { - for (ResourceUpdateMsg resourceUpdateMsg : uplinkMsg.getResourceUpdateMsgList()) { - result.add(((ResourceProcessor) ctx.getResourceEdgeProcessorFactory().getProcessorByEdgeVersion(this.edgeVersion)) - .processResourceMsgFromEdge(edge.getTenantId(), edge, resourceUpdateMsg)); - } - } - if (uplinkMsg.getRuleChainMetadataRequestMsgCount() > 0) { - for (RuleChainMetadataRequestMsg ruleChainMetadataRequestMsg : uplinkMsg.getRuleChainMetadataRequestMsgList()) { - result.add(ctx.getEdgeRequestsService().processRuleChainMetadataRequestMsg(edge.getTenantId(), edge, ruleChainMetadataRequestMsg)); - } - } - if (uplinkMsg.getAttributesRequestMsgCount() > 0) { - for (AttributesRequestMsg attributesRequestMsg : uplinkMsg.getAttributesRequestMsgList()) { - result.add(ctx.getEdgeRequestsService().processAttributesRequestMsg(edge.getTenantId(), edge, attributesRequestMsg)); - } - } - if (uplinkMsg.getRelationRequestMsgCount() > 0) { - for (RelationRequestMsg relationRequestMsg : uplinkMsg.getRelationRequestMsgList()) { - result.add(ctx.getEdgeRequestsService().processRelationRequestMsg(edge.getTenantId(), edge, relationRequestMsg)); - } - } - if (uplinkMsg.getUserCredentialsRequestMsgCount() > 0) { - for (UserCredentialsRequestMsg userCredentialsRequestMsg : uplinkMsg.getUserCredentialsRequestMsgList()) { - result.add(ctx.getEdgeRequestsService().processUserCredentialsRequestMsg(edge.getTenantId(), edge, userCredentialsRequestMsg)); - } - } - if (uplinkMsg.getDeviceCredentialsRequestMsgCount() > 0) { - for (DeviceCredentialsRequestMsg deviceCredentialsRequestMsg : uplinkMsg.getDeviceCredentialsRequestMsgList()) { - result.add(ctx.getEdgeRequestsService().processDeviceCredentialsRequestMsg(edge.getTenantId(), edge, deviceCredentialsRequestMsg)); - } - } - if (uplinkMsg.getDeviceRpcCallMsgCount() > 0) { - for (DeviceRpcCallMsg deviceRpcCallMsg : uplinkMsg.getDeviceRpcCallMsgList()) { - result.add(((DeviceProcessor) ctx.getDeviceEdgeProcessorFactory().getProcessorByEdgeVersion(this.edgeVersion)) - .processDeviceRpcCallFromEdge(edge.getTenantId(), edge, deviceRpcCallMsg)); - } - } - if (uplinkMsg.getWidgetBundleTypesRequestMsgCount() > 0) { - for (WidgetBundleTypesRequestMsg widgetBundleTypesRequestMsg : uplinkMsg.getWidgetBundleTypesRequestMsgList()) { - result.add(ctx.getEdgeRequestsService().processWidgetBundleTypesRequestMsg(edge.getTenantId(), edge, widgetBundleTypesRequestMsg)); - } - } - if (uplinkMsg.getEntityViewsRequestMsgCount() > 0) { - for (EntityViewsRequestMsg entityViewRequestMsg : uplinkMsg.getEntityViewsRequestMsgList()) { - result.add(ctx.getEdgeRequestsService().processEntityViewsRequestMsg(edge.getTenantId(), edge, entityViewRequestMsg)); - } - } - } catch (Exception e) { - String failureMsg = String.format("Can't process uplink msg [%s] from edge", uplinkMsg); - log.error("[{}][{}] Can't process uplink msg [{}]", this.tenantId, this.sessionId, uplinkMsg, e); - ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) - .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(e.getMessage()).build()); - return Futures.immediateFailedFuture(e); - } - return Futures.allAsList(result); - } - - private ConnectResponseMsg processConnect(ConnectRequestMsg request) { - log.trace("[{}] processConnect [{}]", this.sessionId, request); - Optional optional = ctx.getEdgeService().findEdgeByRoutingKey(TenantId.SYS_TENANT_ID, request.getEdgeRoutingKey()); - if (optional.isPresent()) { - edge = optional.get(); - this.tenantId = edge.getTenantId(); - try { - if (edge.getSecret().equals(request.getEdgeSecret())) { - sessionOpenListener.accept(edge.getId(), this); - this.edgeVersion = request.getEdgeVersion(); - processSaveEdgeVersionAsAttribute(request.getEdgeVersion().name()); - return ConnectResponseMsg.newBuilder() - .setResponseCode(ConnectResponseCode.ACCEPTED) - .setErrorMsg("") - .setConfiguration(ctx.getEdgeMsgConstructor().constructEdgeConfiguration(edge)) - .setMaxInboundMessageSize(maxInboundMessageSize) - .build(); - } - String error = "Failed to validate the edge!"; - String failureMsg = String.format("%s Provided request secret: %s", error, request.getEdgeSecret()); - ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) - .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(error).build()); - return ConnectResponseMsg.newBuilder() - .setResponseCode(ConnectResponseCode.BAD_CREDENTIALS) - .setErrorMsg(failureMsg) - .setConfiguration(EdgeConfiguration.getDefaultInstance()).build(); - } catch (Exception e) { - String failureMsg = "Failed to process edge connection!"; - ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) - .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(e.getMessage()).build()); - log.error(failureMsg, e); - return ConnectResponseMsg.newBuilder() - .setResponseCode(ConnectResponseCode.SERVER_UNAVAILABLE) - .setErrorMsg(failureMsg) - .setConfiguration(EdgeConfiguration.getDefaultInstance()).build(); - } - } - return ConnectResponseMsg.newBuilder() - .setResponseCode(ConnectResponseCode.BAD_CREDENTIALS) - .setErrorMsg("Failed to find the edge! Routing key: " + request.getEdgeRoutingKey()) - .setConfiguration(EdgeConfiguration.getDefaultInstance()).build(); - } - - private void processSaveEdgeVersionAsAttribute(String edgeVersion) { - AttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(new StringDataEntry(DataConstants.EDGE_VERSION_ATTR_KEY, edgeVersion), System.currentTimeMillis()); - ctx.getAttributesService().save(this.tenantId, this.edge.getId(), AttributeScope.SERVER_SCOPE, attributeKvEntry); - } - - @Override - public void close() { - log.debug("[{}][{}] Closing session", this.tenantId, sessionId); - connected = false; - try { - outputStream.onCompleted(); - } catch (Exception e) { - log.debug("[{}][{}] Failed to close output stream: {}", this.tenantId, sessionId, e.getMessage()); - } - } - - private void interruptPreviousSendDownlinkMsgsTask() { - if (sessionState.getSendDownlinkMsgsFuture() != null && !sessionState.getSendDownlinkMsgsFuture().isDone() - || sessionState.getScheduledSendDownlinkTask() != null && !sessionState.getScheduledSendDownlinkTask().isCancelled()) { - log.debug("[{}][{}][{}] Previous send downlink future was not properly completed, stopping it now!", this.tenantId, edge.getId(), this.sessionId); - stopCurrentSendDownlinkMsgsTask(true); - } - } - - private void interruptGeneralProcessingOnSync() { - log.debug("[{}][{}][{}] Sync process started. General processing interrupted!", this.tenantId, edge.getId(), this.sessionId); - stopCurrentSendDownlinkMsgsTask(true); - } - - public void stopCurrentSendDownlinkMsgsTask(Boolean isInterrupted) { - if (sessionState.getSendDownlinkMsgsFuture() != null && !sessionState.getSendDownlinkMsgsFuture().isDone()) { - sessionState.getSendDownlinkMsgsFuture().set(isInterrupted); - } - if (sessionState.getScheduledSendDownlinkTask() != null) { - sessionState.getScheduledSendDownlinkTask().cancel(true); - } - } - - public void addEventToHighPriorityQueue(EdgeEvent edgeEvent) { - while (highPriorityQueue.size() > maxHighPriorityQueueSizePerSession) { - EdgeEvent oldestHighPriority = highPriorityQueue.poll(); - if (oldestHighPriority != null) { - log.warn("[{}][{}][{}] High priority queue is full. Removing oldest high priority event from queue {}", - this.tenantId, edge.getId(), this.sessionId, oldestHighPriority); - } - } - highPriorityQueue.add(edgeEvent); - } + boolean isConnected(); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java new file mode 100644 index 0000000000..517b550a36 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.cache.limits.RateLimitService; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.limit.LimitedApi; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.tools.TbRateLimitsException; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.dao.edge.EdgeEventService; +import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; +import org.thingsboard.server.dao.service.DataValidator; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; + +import java.util.Optional; +import java.util.UUID; + +@Slf4j +@Service +@RequiredArgsConstructor +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") +public class KafkaEdgeEventService implements EdgeEventService { + + private final RateLimitService rateLimitService; + private final ApplicationEventPublisher eventPublisher; + private final DataValidator edgeEventValidator; + @Lazy + private final TbQueueProducerProvider producerProvider; + @Lazy + private final TopicService topicService; + private final TbTransactionalCache edgeIdServiceIdCache; + + @Override + public ListenableFuture saveAsync(EdgeEvent edgeEvent) { + if (!rateLimitService.checkRateLimit(LimitedApi.EDGE_EVENTS, edgeEvent.getTenantId())) { + throw new TbRateLimitsException(EntityType.TENANT); + } + if (!rateLimitService.checkRateLimit(LimitedApi.EDGE_EVENTS_PER_EDGE, edgeEvent.getTenantId(), edgeEvent.getEdgeId())) { + throw new TbRateLimitsException(EntityType.EDGE); + } + edgeEventValidator.validate(edgeEvent, EdgeEvent::getTenantId); + var serviceIdOpt = Optional.ofNullable(edgeIdServiceIdCache.get(edgeEvent.getEdgeId())); + TopicPartitionInfo tpi = topicService.getEdgeEventNotificationsTopic(edgeEvent.getTenantId(), edgeEvent.getEdgeId(), serviceIdOpt.get().get()); + TransportProtos.ToEdgeEventNotificationMsg msg = TransportProtos.ToEdgeEventNotificationMsg.newBuilder().setEdgeEventMsg(ProtoUtils.toProto(edgeEvent)).build(); + producerProvider.getTbEdgeEventsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); + + eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(edgeEvent.getTenantId()).entity(edgeEvent).entityId(edgeEvent.getEdgeId()).build()); + return Futures.immediateFuture(null); + } + + @Override + public PageData findEdgeEvents(TenantId tenantId, EdgeId edgeId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink) { + return null; + } + + @Override + public void cleanupEvents(long ttl) { + + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java new file mode 100644 index 0000000000..78acd02d24 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java @@ -0,0 +1,112 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import io.grpc.stub.StreamObserver; +import jakarta.annotation.PreDestroy; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.edge.v1.DownlinkMsg; +import org.thingsboard.server.gen.edge.v1.ResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.edge.EdgeContextComponent; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.BiConsumer; + +@Slf4j +public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession { + + private TbQueueConsumer> edgeEventsConsumer; + + public KafkaEdgeGrpcSession(EdgeContextComponent ctx, StreamObserver outputStream, + BiConsumer sessionOpenListener, + BiConsumer sessionCloseListener, + ScheduledExecutorService sendDownlinkExecutorService, + int maxInboundMessageSize, int maxHighPriorityQueueSizePerSession) { + super(ctx, outputStream, sessionOpenListener, sessionCloseListener, sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession); + } + + @Override + protected ListenableFuture processEdgeEvents() { + log.trace("[{}][{}] starting processing edge events", this.tenantId, this.sessionId); + if (isConnected() && isSyncCompleted()) { + + if (!highPriorityQueue.isEmpty()) { + processHighPriorityEvents(); + } else { + return processKafkaEvents(); + } + } else { + log.trace("[{}][{}] edge is not connected or sync is not completed. Skipping iteration", tenantId, sessionId); + } + return null; + } + + private ListenableFuture processKafkaEvents() { + List edgeEvents = new ArrayList<>(); + try { + edgeEventsConsumer.subscribe(); + List> messages = edgeEventsConsumer.poll(100); + + if (messages.isEmpty()) { + return Futures.immediateFuture(Boolean.FALSE); + } + + for (TbProtoQueueMsg msg : messages) { + EdgeEvent edgeEvent = ProtoUtils.fromProto(msg.getValue().getEdgeEventMsg()); + edgeEvents.add(edgeEvent); + } + + List downlinkMsgsPack = convertToDownlinkMsgsPack(edgeEvents); + sendDownlinkMsgsPack(downlinkMsgsPack).get(); + edgeEventsConsumer.commit(); + return Futures.immediateFuture(Boolean.TRUE); + } catch (Exception e) { + log.error("[{}][{}] Error occurred while polling edge events from Kafka: {}", tenantId, edge.getId(), e.getMessage()); + return Futures.immediateFailedFuture(e); + } + } + + protected void initConsumer(TbQueueConsumer> edgeEventsConsumer) { + this.edgeEventsConsumer = edgeEventsConsumer; + } + + @PreDestroy + private void destroy() { + if (edgeEventsConsumer != null) { + edgeEventsConsumer.unsubscribe(); + } + } + + public void stopConsumer() { + if (edgeEventsConsumer != null) { + edgeEventsConsumer.stop(); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java new file mode 100644 index 0000000000..47fad107a1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java @@ -0,0 +1,187 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.springframework.data.util.Pair; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.SortOrder; +import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.gen.edge.v1.ResponseMsg; +import org.thingsboard.server.service.edge.EdgeContextComponent; +import org.thingsboard.server.service.edge.rpc.fetch.GeneralEdgeEventFetcher; + +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.BiConsumer; + +@Slf4j +public class PostgresEdgeGrpcSession extends AbstractEdgeGrpcSession { + + PostgresEdgeGrpcSession(EdgeContextComponent ctx, StreamObserver outputStream, + BiConsumer sessionOpenListener, + BiConsumer sessionCloseListener, ScheduledExecutorService sendDownlinkExecutorService, + int maxInboundMessageSize, int maxHighPriorityQueueSizePerSession) { + super(ctx, outputStream, sessionOpenListener, sessionCloseListener, sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession); + initInputStream(); + } + + @Override + protected ListenableFuture processEdgeEvents() throws Exception { + SettableFuture result = SettableFuture.create(); + log.trace("[{}][{}] starting processing edge events", this.tenantId, this.sessionId); + if (isConnected() && isSyncCompleted()) { + Pair startTsAndSeqId = getQueueStartTsAndSeqId().get(); + this.previousStartTs = startTsAndSeqId.getFirst(); + this.previousStartSeqId = startTsAndSeqId.getSecond(); + GeneralEdgeEventFetcher fetcher = new GeneralEdgeEventFetcher( + this.previousStartTs, + this.previousStartSeqId, + this.seqIdEnd, + false, + Integer.toUnsignedLong(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount()), + ctx.getEdgeEventService()); + Futures.addCallback(startProcessingEdgeEvents(fetcher), new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Pair newStartTsAndSeqId) { + if (newStartTsAndSeqId != null) { + ListenableFuture> updateFuture = updateQueueStartTsAndSeqId(newStartTsAndSeqId); + Futures.addCallback(updateFuture, new FutureCallback<>() { + @Override + public void onSuccess(@Nullable List list) { + log.debug("[{}][{}] queue offset was updated [{}]", tenantId, sessionId, newStartTsAndSeqId); + if (fetcher.isSeqIdNewCycleStarted()) { + seqIdEnd = fetcher.getSeqIdEnd(); + boolean newEventsAvailable = isNewEdgeEventsAvailable(); + result.set(newEventsAvailable); + } else { + seqIdEnd = null; + boolean newEventsAvailable = isSeqIdStartedNewCycle(); + if (!newEventsAvailable) { + newEventsAvailable = isNewEdgeEventsAvailable(); + } + result.set(newEventsAvailable); + } + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}][{}] Failed to update queue offset [{}]", tenantId, sessionId, newStartTsAndSeqId, t); + result.setException(t); + } + }, ctx.getGrpcCallbackExecutorService()); + } else { + log.trace("[{}][{}] newStartTsAndSeqId is null. Skipping iteration without db update", tenantId, sessionId); + result.set(null); + } + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}][{}] Failed to process events", tenantId, sessionId, t); + result.setException(t); + } + }, ctx.getGrpcCallbackExecutorService()); + } else { + log.trace("[{}][{}] edge is not connected or sync is not completed. Skipping iteration", tenantId, sessionId); + result.set(null); + } + return result; + } + + private ListenableFuture> getQueueStartTsAndSeqId() { + ListenableFuture> future = + ctx.getAttributesService().find(edge.getTenantId(), edge.getId(), AttributeScope.SERVER_SCOPE, Arrays.asList(QUEUE_START_TS_ATTR_KEY, QUEUE_START_SEQ_ID_ATTR_KEY)); + return Futures.transform(future, attributeKvEntries -> { + long startTs = 0L; + long startSeqId = 0L; + for (AttributeKvEntry attributeKvEntry : attributeKvEntries) { + if (QUEUE_START_TS_ATTR_KEY.equals(attributeKvEntry.getKey())) { + startTs = attributeKvEntry.getLongValue().isPresent() ? attributeKvEntry.getLongValue().get() : 0L; + } + if (QUEUE_START_SEQ_ID_ATTR_KEY.equals(attributeKvEntry.getKey())) { + startSeqId = attributeKvEntry.getLongValue().isPresent() ? attributeKvEntry.getLongValue().get() : 0L; + } + } + if (startSeqId == 0L) { + startSeqId = findStartSeqIdFromOldestEventIfAny(); + } + return Pair.of(startTs, startSeqId); + }, ctx.getGrpcCallbackExecutorService()); + } + + private boolean isSeqIdStartedNewCycle() { + try { + TimePageLink pageLink = new TimePageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount(), 0, null, null, this.newStartTs, System.currentTimeMillis()); + PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), 0L, this.previousStartSeqId == 0 ? null : this.previousStartSeqId - 1, pageLink); + return !edgeEvents.getData().isEmpty(); + } catch (Exception e) { + log.error("[{}][{}][{}] Failed to execute isSeqIdStartedNewCycle", this.tenantId, edge.getId(), sessionId, e); + } + return false; + } + + private boolean isNewEdgeEventsAvailable() { + try { + TimePageLink pageLink = new TimePageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount(), 0, null, null, this.newStartTs, System.currentTimeMillis()); + PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), this.newStartSeqId, null, pageLink); + return !edgeEvents.getData().isEmpty() || !highPriorityQueue.isEmpty(); + } catch (Exception e) { + log.error("[{}][{}][{}] Failed to execute isNewEdgeEventsAvailable", this.tenantId, edge.getId(), sessionId, e); + } + return false; + } + + private long findStartSeqIdFromOldestEventIfAny() { + long startSeqId = 0L; + try { + TimePageLink pageLink = new TimePageLink(1, 0, null, new SortOrder("createdTime"), null, null); + PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), null, null, pageLink); + if (!edgeEvents.getData().isEmpty()) { + startSeqId = edgeEvents.getData().get(0).getSeqId() - 1; + } + } catch (Exception e) { + log.error("[{}][{}][{}] Failed to execute findStartSeqIdFromOldestEventIfAny", this.tenantId, edge.getId(), sessionId, e); + } + return startSeqId; + } + + private ListenableFuture> updateQueueStartTsAndSeqId(Pair pair) { + this.newStartTs = pair.getFirst(); + this.newStartSeqId = pair.getSecond(); + log.trace("[{}] updateQueueStartTsAndSeqId [{}][{}][{}]", this.sessionId, edge.getId(), this.newStartTs, this.newStartSeqId); + List attributes = Arrays.asList( + new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_TS_ATTR_KEY, this.newStartTs), System.currentTimeMillis()), + new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_SEQ_ID_ATTR_KEY, this.newStartSeqId), System.currentTimeMillis())); + return ctx.getAttributesService().save(edge.getTenantId(), edge.getId(), AttributeScope.SERVER_SCOPE, attributes); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index 3c18e02324..7c823a0dfa 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -22,17 +22,10 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.Dashboard; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.TbResource; -import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; @@ -59,72 +52,17 @@ import org.thingsboard.server.common.data.rule.RuleChainConnectionInfo; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.dao.alarm.AlarmCommentService; -import org.thingsboard.server.dao.alarm.AlarmService; -import org.thingsboard.server.dao.asset.AssetProfileService; -import org.thingsboard.server.dao.asset.AssetService; -import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.customer.CustomerService; -import org.thingsboard.server.dao.dashboard.DashboardService; -import org.thingsboard.server.dao.device.DeviceCredentialsService; -import org.thingsboard.server.dao.device.DeviceProfileService; -import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.dao.domain.DomainService; -import org.thingsboard.server.dao.edge.EdgeEventService; -import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.edge.EdgeSynchronizationManager; -import org.thingsboard.server.dao.entityview.EntityViewService; -import org.thingsboard.server.dao.notification.NotificationRuleService; -import org.thingsboard.server.dao.notification.NotificationTargetService; -import org.thingsboard.server.dao.notification.NotificationTemplateService; -import org.thingsboard.server.dao.oauth2.OAuth2ClientService; -import org.thingsboard.server.dao.ota.OtaPackageService; -import org.thingsboard.server.dao.queue.QueueService; -import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.dao.resource.ResourceService; -import org.thingsboard.server.dao.rule.RuleChainService; -import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantProfileService; -import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.dao.user.UserService; -import org.thingsboard.server.dao.widget.WidgetTypeService; -import org.thingsboard.server.dao.widget.WidgetsBundleService; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.service.edge.rpc.constructor.alarm.AlarmMsgConstructorFactory; -import org.thingsboard.server.service.edge.rpc.constructor.asset.AssetMsgConstructorFactory; -import org.thingsboard.server.service.edge.rpc.constructor.customer.CustomerMsgConstructorFactory; -import org.thingsboard.server.service.edge.rpc.constructor.dashboard.DashboardMsgConstructorFactory; -import org.thingsboard.server.service.edge.rpc.constructor.device.DeviceMsgConstructorFactory; -import org.thingsboard.server.service.edge.rpc.constructor.edge.EdgeMsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.entityview.EntityViewMsgConstructorFactory; -import org.thingsboard.server.service.edge.rpc.constructor.notification.NotificationMsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.oauth2.OAuth2MsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.ota.OtaPackageMsgConstructorFactory; -import org.thingsboard.server.service.edge.rpc.constructor.queue.QueueMsgConstructorFactory; -import org.thingsboard.server.service.edge.rpc.constructor.relation.RelationMsgConstructorFactory; -import org.thingsboard.server.service.edge.rpc.constructor.resource.ResourceMsgConstructorFactory; -import org.thingsboard.server.service.edge.rpc.constructor.rule.RuleChainMsgConstructorFactory; -import org.thingsboard.server.service.edge.rpc.constructor.settings.AdminSettingsMsgConstructorFactory; -import org.thingsboard.server.service.edge.rpc.constructor.telemetry.EntityDataMsgConstructor; -import org.thingsboard.server.service.edge.rpc.constructor.tenant.TenantMsgConstructorFactory; -import org.thingsboard.server.service.edge.rpc.constructor.user.UserMsgConstructorFactory; -import org.thingsboard.server.service.edge.rpc.constructor.widget.WidgetMsgConstructorFactory; -import org.thingsboard.server.service.edge.rpc.processor.alarm.AlarmEdgeProcessorFactory; -import org.thingsboard.server.service.edge.rpc.processor.asset.AssetEdgeProcessorFactory; -import org.thingsboard.server.service.edge.rpc.processor.entityview.EntityViewProcessorFactory; -import org.thingsboard.server.service.entitiy.TbLogEntityActionService; +import org.thingsboard.server.service.edge.EdgeContextComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; -import org.thingsboard.server.service.profile.TbAssetProfileCache; -import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.state.DefaultDeviceStateService; -import org.thingsboard.server.service.state.DeviceStateService; -import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.util.ArrayList; import java.util.List; @@ -142,201 +80,7 @@ public abstract class BaseEdgeProcessor { protected static final Lock assetCreationLock = new ReentrantLock(); @Autowired - protected TelemetrySubscriptionService tsSubService; - - @Autowired - protected TbLogEntityActionService logEntityActionService; - - @Autowired - protected RuleChainService ruleChainService; - - @Autowired - protected AlarmService alarmService; - - @Autowired - protected AlarmCommentService alarmCommentService; - - @Autowired - protected DeviceService deviceService; - - @Autowired - protected TbDeviceProfileCache deviceProfileCache; - - @Autowired - protected TbAssetProfileCache assetProfileCache; - - @Autowired - protected DashboardService dashboardService; - - @Autowired - protected AssetService assetService; - - @Autowired - protected EntityViewService entityViewService; - - @Autowired - protected TenantService tenantService; - - @Autowired - protected TenantProfileService tenantProfileService; - - @Autowired - protected EdgeService edgeService; - - @Autowired - protected CustomerService customerService; - - @Autowired - protected UserService userService; - - @Autowired - protected DeviceProfileService deviceProfileService; - - @Autowired - protected AssetProfileService assetProfileService; - - @Autowired - protected RelationService relationService; - - @Autowired - protected DeviceCredentialsService deviceCredentialsService; - - @Autowired - protected AttributesService attributesService; - - @Autowired - protected TbClusterService tbClusterService; - - @Autowired - protected DeviceStateService deviceStateService; - - @Autowired - protected EdgeEventService edgeEventService; - - @Autowired - protected WidgetsBundleService widgetsBundleService; - - @Autowired - protected WidgetTypeService widgetTypeService; - - @Autowired - protected OtaPackageService otaPackageService; - - @Autowired - protected QueueService queueService; - - @Autowired - protected PartitionService partitionService; - - @Autowired - protected ResourceService resourceService; - - @Autowired - protected NotificationRuleService notificationRuleService; - - @Autowired - protected NotificationTargetService notificationTargetService; - - @Autowired - protected NotificationTemplateService notificationTemplateService; - - @Autowired - protected OAuth2ClientService oAuth2ClientService; - - @Autowired - protected DomainService domainService; - - @Autowired - @Lazy - protected TbQueueProducerProvider producerProvider; - - @Autowired - protected DataValidator deviceValidator; - - @Autowired - protected DataValidator deviceProfileValidator; - - @Autowired - protected DataValidator assetValidator; - - @Autowired - protected DataValidator assetProfileValidator; - - @Autowired - protected DataValidator dashboardValidator; - - @Autowired - protected DataValidator entityViewValidator; - - @Autowired - protected DataValidator resourceValidator; - - @Autowired - protected EdgeMsgConstructor edgeMsgConstructor; - - @Autowired - protected EntityDataMsgConstructor entityDataMsgConstructor; - - @Autowired - protected NotificationMsgConstructor notificationMsgConstructor; - - @Autowired - protected OAuth2MsgConstructor oAuth2MsgConstructor; - - - @Autowired - protected RuleChainMsgConstructorFactory ruleChainMsgConstructorFactory; - - @Autowired - protected AlarmMsgConstructorFactory alarmMsgConstructorFactory; - - @Autowired - protected DeviceMsgConstructorFactory deviceMsgConstructorFactory; - - @Autowired - protected AssetMsgConstructorFactory assetMsgConstructorFactory; - - @Autowired - protected EntityViewMsgConstructorFactory entityViewMsgConstructorFactory; - - @Autowired - protected DashboardMsgConstructorFactory dashboardMsgConstructorFactory; - - @Autowired - protected RelationMsgConstructorFactory relationMsgConstructorFactory; - - @Autowired - protected UserMsgConstructorFactory userMsgConstructorFactory; - - @Autowired - protected CustomerMsgConstructorFactory customerMsgConstructorFactory; - - @Autowired - protected TenantMsgConstructorFactory tenantMsgConstructorFactory; - - @Autowired - protected WidgetMsgConstructorFactory widgetMsgConstructorFactory; - - @Autowired - protected AdminSettingsMsgConstructorFactory adminSettingsMsgConstructorFactory; - - @Autowired - protected OtaPackageMsgConstructorFactory otaPackageMsgConstructorFactory; - - @Autowired - protected QueueMsgConstructorFactory queueMsgConstructorFactory; - - @Autowired - protected ResourceMsgConstructorFactory resourceMsgConstructorFactory; - - @Autowired - protected AlarmEdgeProcessorFactory alarmEdgeProcessorFactory; - - @Autowired - protected AssetEdgeProcessorFactory assetEdgeProcessorFactory; - - @Autowired - protected EntityViewProcessorFactory entityViewProcessorFactory; + protected EdgeContextComponent edgeCtx; @Autowired protected EdgeSynchronizationManager edgeSynchronizationManager; @@ -344,6 +88,9 @@ public abstract class BaseEdgeProcessor { @Autowired protected DbCallbackExecutorService dbCallbackExecutorService; + @Autowired + private TbTransactionalCache edgeIdServiceIdCache; + protected ListenableFuture saveEdgeEvent(TenantId tenantId, EdgeId edgeId, EdgeEventType type, @@ -351,7 +98,7 @@ public abstract class BaseEdgeProcessor { EntityId entityId, JsonNode body) { ListenableFuture> future = - attributesService.find(tenantId, edgeId, AttributeScope.SERVER_SCOPE, DefaultDeviceStateService.ACTIVITY_STATE); + edgeCtx.getAttributesService().find(tenantId, edgeId, AttributeScope.SERVER_SCOPE, DefaultDeviceStateService.ACTIVITY_STATE); return Futures.transformAsync(future, activeOpt -> { if (activeOpt.isEmpty()) { log.trace("Edge is not activated. Skipping event. tenantId [{}], edgeId [{}], type[{}], " + @@ -376,11 +123,13 @@ public abstract class BaseEdgeProcessor { private boolean doSaveIfEdgeIsOffline(EdgeEventType type, EdgeEventActionType action) { return switch (action) { - case TIMESERIES_UPDATED, ALARM_ACK, ALARM_CLEAR, ALARM_ASSIGNED, ALARM_UNASSIGNED, ADDED_COMMENT, UPDATED_COMMENT -> - true; + case TIMESERIES_UPDATED, ALARM_ACK, ALARM_CLEAR, ALARM_ASSIGNED, ALARM_UNASSIGNED, ADDED_COMMENT, + UPDATED_COMMENT -> true; default -> switch (type) { - case ALARM, ALARM_COMMENT, RULE_CHAIN, RULE_CHAIN_METADATA, USER, CUSTOMER, TENANT, TENANT_PROFILE, WIDGETS_BUNDLE, WIDGET_TYPE, - ADMIN_SETTINGS, OTA_PACKAGE, QUEUE, RELATION, NOTIFICATION_TEMPLATE, NOTIFICATION_TARGET, NOTIFICATION_RULE -> true; + case ALARM, ALARM_COMMENT, RULE_CHAIN, RULE_CHAIN_METADATA, USER, CUSTOMER, TENANT, TENANT_PROFILE, + WIDGETS_BUNDLE, WIDGET_TYPE, + ADMIN_SETTINGS, OTA_PACKAGE, QUEUE, RELATION, NOTIFICATION_TEMPLATE, NOTIFICATION_TARGET, + NOTIFICATION_RULE -> true; default -> false; }; }; @@ -391,8 +140,7 @@ public abstract class BaseEdgeProcessor { tenantId, edgeId, type, action, entityId, body); EdgeEvent edgeEvent = EdgeUtils.constructEdgeEvent(tenantId, edgeId, type, action, entityId, body); - - return edgeEventService.saveAsync(edgeEvent); + return edgeCtx.getEdgeEventService().saveAsync(edgeEvent); } protected ListenableFuture processActionForAllEdges(TenantId tenantId, EdgeEventType type, @@ -400,7 +148,7 @@ public abstract class BaseEdgeProcessor { JsonNode body, EdgeId sourceEdgeId) { List> futures = new ArrayList<>(); if (TenantId.SYS_TENANT_ID.equals(tenantId)) { - PageDataIterable tenantIds = new PageDataIterable<>(link -> tenantService.findTenantsIds(link), 1024); + PageDataIterable tenantIds = new PageDataIterable<>(link -> edgeCtx.getTenantService().findTenantsIds(link), 1024); for (TenantId tenantId1 : tenantIds) { futures.addAll(processActionForAllEdgesByTenantId(tenantId1, type, actionType, entityId, body, sourceEdgeId)); } @@ -417,7 +165,7 @@ public abstract class BaseEdgeProcessor { JsonNode body, EdgeId sourceEdgeId) { List> futures = new ArrayList<>(); - PageDataIterable edges = new PageDataIterable<>(link -> edgeService.findEdgesByTenantId(tenantId, link), 1024); + PageDataIterable edges = new PageDataIterable<>(link -> edgeCtx.getEdgeService().findEdgesByTenantId(tenantId, link), 1024); for (Edge edge : edges) { if (!edge.getId().equals(sourceEdgeId)) { futures.add(saveEdgeEvent(tenantId, edge.getId(), type, actionType, entityId, body)); @@ -436,8 +184,10 @@ public abstract class BaseEdgeProcessor { return switch (actionType) { case UPDATED, CREDENTIALS_UPDATED, ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER, UPDATED_COMMENT -> UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE; - case ADDED, ASSIGNED_TO_EDGE, RELATION_ADD_OR_UPDATE, ADDED_COMMENT -> UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE; - case DELETED, UNASSIGNED_FROM_EDGE, RELATION_DELETED, DELETED_COMMENT, ALARM_DELETE -> UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE; + case ADDED, ASSIGNED_TO_EDGE, RELATION_ADD_OR_UPDATE, ADDED_COMMENT -> + UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE; + case DELETED, UNASSIGNED_FROM_EDGE, RELATION_DELETED, DELETED_COMMENT, ALARM_DELETE -> + UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE; case ALARM_ACK -> UpdateMsgType.ALARM_ACK_RPC_MESSAGE; case ALARM_CLEAR -> UpdateMsgType.ALARM_CLEAR_RPC_MESSAGE; default -> throw new RuntimeException("Unsupported actionType [" + actionType + "]"); @@ -504,7 +254,7 @@ public abstract class BaseEdgeProcessor { EdgeEventActionType actionType, EdgeId sourceEdgeId) { List> futures = new ArrayList<>(); PageDataIterableByTenantIdEntityId edgeIds = - new PageDataIterableByTenantIdEntityId<>(edgeService::findRelatedEdgeIdsByEntityId, tenantId, entityId, RELATED_EDGES_CACHE_ITEMS); + new PageDataIterableByTenantIdEntityId<>(edgeCtx.getEdgeService()::findRelatedEdgeIdsByEntityId, tenantId, entityId, RELATED_EDGES_CACHE_ITEMS); for (EdgeId relatedEdgeId : edgeIds) { if (!relatedEdgeId.equals(sourceEdgeId)) { futures.add(saveEdgeEvent(tenantId, relatedEdgeId, type, actionType, entityId, null)); @@ -515,10 +265,10 @@ public abstract class BaseEdgeProcessor { private ListenableFuture updateDependentRuleChains(TenantId tenantId, RuleChainId processingRuleChainId, EdgeId edgeId) { List> futures = new ArrayList<>(); - PageDataIterable ruleChains = new PageDataIterable<>(link -> ruleChainService.findRuleChainsByTenantIdAndEdgeId(tenantId, edgeId, link), 1024); + PageDataIterable ruleChains = new PageDataIterable<>(link -> edgeCtx.getRuleChainService().findRuleChainsByTenantIdAndEdgeId(tenantId, edgeId, link), 1024); for (RuleChain ruleChain : ruleChains) { List connectionInfos = - ruleChainService.loadRuleChainMetaData(ruleChain.getTenantId(), ruleChain.getId()).getRuleChainConnections(); + edgeCtx.getRuleChainService().loadRuleChainMetaData(ruleChain.getTenantId(), ruleChain.getId()).getRuleChainConnections(); if (connectionInfos != null && !connectionInfos.isEmpty()) { for (RuleChainConnectionInfo connectionInfo : connectionInfos) { if (connectionInfo.getTargetRuleChainId().equals(processingRuleChainId)) { @@ -577,14 +327,17 @@ public abstract class BaseEdgeProcessor { protected boolean isEntityExists(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { - case TENANT -> tenantService.findTenantById(tenantId) != null; - case DEVICE -> deviceService.findDeviceById(tenantId, new DeviceId(entityId.getId())) != null; - case ASSET -> assetService.findAssetById(tenantId, new AssetId(entityId.getId())) != null; - case ENTITY_VIEW -> entityViewService.findEntityViewById(tenantId, new EntityViewId(entityId.getId())) != null; - case CUSTOMER -> customerService.findCustomerById(tenantId, new CustomerId(entityId.getId())) != null; - case USER -> userService.findUserById(tenantId, new UserId(entityId.getId())) != null; - case DASHBOARD -> dashboardService.findDashboardById(tenantId, new DashboardId(entityId.getId())) != null; - case EDGE -> edgeService.findEdgeById(tenantId, new EdgeId(entityId.getId())) != null; + case TENANT -> edgeCtx.getTenantService().findTenantById(tenantId) != null; + case DEVICE -> edgeCtx.getDeviceService().findDeviceById(tenantId, new DeviceId(entityId.getId())) != null; + case ASSET -> edgeCtx.getAssetService().findAssetById(tenantId, new AssetId(entityId.getId())) != null; + case ENTITY_VIEW -> + edgeCtx.getEntityViewService().findEntityViewById(tenantId, new EntityViewId(entityId.getId())) != null; + case CUSTOMER -> + edgeCtx.getCustomerService().findCustomerById(tenantId, new CustomerId(entityId.getId())) != null; + case USER -> edgeCtx.getUserService().findUserById(tenantId, new UserId(entityId.getId())) != null; + case DASHBOARD -> + edgeCtx.getDashboardService().findDashboardById(tenantId, new DashboardId(entityId.getId())) != null; + case EDGE -> edgeCtx.getEdgeService().findEdgeById(tenantId, new EdgeId(entityId.getId())) != null; default -> false; }; } @@ -595,7 +348,7 @@ public abstract class BaseEdgeProcessor { relation.setTo(entityId); relation.setTypeGroup(RelationTypeGroup.COMMON); relation.setType(EntityRelation.EDGE_TYPE); - relationService.saveRelation(tenantId, relation); + edgeCtx.getRelationService().saveRelation(tenantId, relation); } protected TbMsgMetaData getEdgeActionTbMsgMetaData(Edge edge, CustomerId customerId) { @@ -611,7 +364,7 @@ public abstract class BaseEdgeProcessor { protected void pushEntityEventToRuleEngine(TenantId tenantId, EntityId entityId, CustomerId customerId, TbMsgType msgType, String msgData, TbMsgMetaData metaData) { TbMsg tbMsg = TbMsg.newMsg(msgType, entityId, customerId, metaData, TbMsgDataType.JSON, msgData); - tbClusterService.pushMsgToRuleEngine(tenantId, entityId, tbMsg, new TbQueueCallback() { + edgeCtx.getClusterService().pushMsgToRuleEngine(tenantId, entityId, tbMsg, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { log.debug("[{}] Successfully send ENTITY_CREATED EVENT to rule engine [{}]", tenantId, msgData); @@ -624,39 +377,4 @@ public abstract class BaseEdgeProcessor { }); } - protected AssetProfile checkIfAssetProfileDefaultFieldsAssignedToEdge(TenantId tenantId, EdgeId edgeId, AssetProfile assetProfile, EdgeVersion edgeVersion) { - if (EdgeVersion.V_3_3_0.equals(edgeVersion) || EdgeVersion.V_3_3_3.equals(edgeVersion) || EdgeVersion.V_3_4_0.equals(edgeVersion)) { - if (assetProfile.getDefaultDashboardId() != null && isEntityNotAssignedToEdge(tenantId, assetProfile.getDefaultDashboardId(), edgeId)) { - assetProfile.setDefaultDashboardId(null); - } - if (assetProfile.getDefaultEdgeRuleChainId() != null && isEntityNotAssignedToEdge(tenantId, assetProfile.getDefaultEdgeRuleChainId(), edgeId)) { - assetProfile.setDefaultEdgeRuleChainId(null); - } - } - return assetProfile; - } - - protected DeviceProfile checkIfDeviceProfileDefaultFieldsAssignedToEdge(TenantId tenantId, EdgeId edgeId, DeviceProfile deviceProfile, EdgeVersion edgeVersion) { - if (EdgeVersion.V_3_3_0.equals(edgeVersion) || EdgeVersion.V_3_3_3.equals(edgeVersion) || EdgeVersion.V_3_4_0.equals(edgeVersion)) { - if (deviceProfile.getDefaultDashboardId() != null && isEntityNotAssignedToEdge(tenantId, deviceProfile.getDefaultDashboardId(), edgeId)) { - deviceProfile.setDefaultDashboardId(null); - } - if (deviceProfile.getDefaultEdgeRuleChainId() != null && isEntityNotAssignedToEdge(tenantId, deviceProfile.getDefaultEdgeRuleChainId(), edgeId)) { - deviceProfile.setDefaultEdgeRuleChainId(null); - } - } - return deviceProfile; - } - - private boolean isEntityNotAssignedToEdge(TenantId tenantId, EntityId entityId, EdgeId edgeId) { - PageDataIterableByTenantIdEntityId edgeIds = - new PageDataIterableByTenantIdEntityId<>(edgeService::findRelatedEdgeIdsByEntityId, tenantId, entityId, RELATED_EDGES_CACHE_ITEMS); - for (EdgeId edgeId1 : edgeIds) { - if (edgeId1.equals(edgeId)) { - return false; - } - } - return true; - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java index 009c5854cd..650aafb8cc 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java @@ -19,16 +19,23 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EdgeUtils; +import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmComment; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterableByTenantIdEntityId; import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; @@ -38,6 +45,7 @@ import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.edge.rpc.constructor.alarm.AlarmMsgConstructor; +import org.thingsboard.server.service.edge.rpc.constructor.alarm.AlarmMsgConstructorFactory; import java.util.ArrayList; import java.util.List; @@ -48,6 +56,9 @@ import static org.thingsboard.server.dao.edge.BaseRelatedEdgesService.RELATED_ED @Slf4j public abstract class AlarmEdgeProcessor extends BaseAlarmProcessor implements AlarmProcessor { + @Autowired + private AlarmMsgConstructorFactory alarmMsgConstructorFactory; + @Override public ListenableFuture processAlarmMsgFromEdge(TenantId tenantId, EdgeId edgeId, AlarmUpdateMsg alarmUpdateMsg) { log.trace("[{}] processAlarmMsgFromEdge [{}]", tenantId, alarmUpdateMsg); @@ -86,6 +97,7 @@ public abstract class AlarmEdgeProcessor extends BaseAlarmProcessor implements A @Override public DownlinkMsg convertAlarmCommentEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); + var msgConstructor = (AlarmMsgConstructor) alarmMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED_COMMENT: case UPDATED_COMMENT: @@ -94,8 +106,7 @@ public abstract class AlarmEdgeProcessor extends BaseAlarmProcessor implements A if (alarmComment != null) { return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addAlarmCommentUpdateMsg(((AlarmMsgConstructor) alarmMsgConstructorFactory - .getMsgConstructorByEdgeVersion(edgeVersion)).constructAlarmCommentUpdatedMsg(msgType, alarmComment)) + .addAlarmCommentUpdateMsg(msgConstructor.constructAlarmCommentUpdatedMsg(msgType, alarmComment)) .build(); } default: @@ -116,7 +127,7 @@ public abstract class AlarmEdgeProcessor extends BaseAlarmProcessor implements A alarmId, actionType, JacksonUtil.valueToTree(deletedAlarm), originatorEdgeId, EdgeEventType.ALARM); return Futures.transform(Futures.allAsList(delFutures), voids -> null, dbCallbackExecutorService); } - ListenableFuture alarmFuture = alarmService.findAlarmByIdAsync(tenantId, alarmId); + ListenableFuture alarmFuture = edgeCtx.getAlarmService().findAlarmByIdAsync(tenantId, alarmId); return Futures.transformAsync(alarmFuture, alarm -> { if (alarm == null) { return Futures.immediateFuture(null); @@ -139,7 +150,7 @@ public abstract class AlarmEdgeProcessor extends BaseAlarmProcessor implements A if (alarmComment == null) { return Futures.immediateFuture(null); } - Alarm alarmById = alarmService.findAlarmById(tenantId, new AlarmId(alarmComment.getAlarmId().getId())); + Alarm alarmById = edgeCtx.getAlarmService().findAlarmById(tenantId, new AlarmId(alarmComment.getAlarmId().getId())); List> delFutures = pushEventToAllRelatedEdges(tenantId, alarmById.getOriginator(), alarmId, actionType, JacksonUtil.valueToTree(alarmComment), originatorEdgeId, EdgeEventType.ALARM_COMMENT); return Futures.transform(Futures.allAsList(delFutures), voids -> null, dbCallbackExecutorService); @@ -150,7 +161,7 @@ public abstract class AlarmEdgeProcessor extends BaseAlarmProcessor implements A EdgeEventType edgeEventType) { List> futures = new ArrayList<>(); PageDataIterableByTenantIdEntityId edgeIds = - new PageDataIterableByTenantIdEntityId<>(edgeService::findRelatedEdgeIdsByEntityId, tenantId, originatorId, RELATED_EDGES_CACHE_ITEMS); + new PageDataIterableByTenantIdEntityId<>(edgeCtx.getEdgeService()::findRelatedEdgeIdsByEntityId, tenantId, originatorId, RELATED_EDGES_CACHE_ITEMS); for (EdgeId relatedEdgeId : edgeIds) { if (!relatedEdgeId.equals(sourceEdgeId)) { futures.add(saveEdgeEvent(tenantId, relatedEdgeId, edgeEventType, actionType, alarmId, body)); @@ -159,4 +170,50 @@ public abstract class AlarmEdgeProcessor extends BaseAlarmProcessor implements A return futures; } + private AlarmUpdateMsg convertAlarmEventToAlarmMsg(TenantId tenantId, UUID entityId, EdgeEventActionType actionType, JsonNode body, EdgeVersion edgeVersion) { + AlarmId alarmId = new AlarmId(entityId); + UpdateMsgType msgType = getUpdateMsgType(actionType); + var msgConstructor = (AlarmMsgConstructor) alarmMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); + switch (actionType) { + case ADDED, UPDATED, ALARM_ACK, ALARM_CLEAR -> { + Alarm alarm = edgeCtx.getAlarmService().findAlarmById(tenantId, alarmId); + if (alarm != null) { + return msgConstructor.constructAlarmUpdatedMsg(msgType, alarm, findOriginatorEntityName(tenantId, alarm)); + } + } + case ALARM_DELETE, DELETED -> { + Alarm deletedAlarm = JacksonUtil.convertValue(body, Alarm.class); + if (deletedAlarm != null) { + return msgConstructor.constructAlarmUpdatedMsg(msgType, deletedAlarm, findOriginatorEntityName(tenantId, deletedAlarm)); + } + } + } + return null; + } + + private String findOriginatorEntityName(TenantId tenantId, Alarm alarm) { + String entityName = null; + switch (alarm.getOriginator().getEntityType()) { + case DEVICE -> { + Device deviceById = edgeCtx.getDeviceService().findDeviceById(tenantId, new DeviceId(alarm.getOriginator().getId())); + if (deviceById != null) { + entityName = deviceById.getName(); + } + } + case ASSET -> { + Asset assetById = edgeCtx.getAssetService().findAssetById(tenantId, new AssetId(alarm.getOriginator().getId())); + if (assetById != null) { + entityName = assetById.getName(); + } + } + case ENTITY_VIEW -> { + EntityView entityViewById = edgeCtx.getEntityViewService().findEntityViewById(tenantId, new EntityViewId(alarm.getOriginator().getId())); + if (entityViewById != null) { + entityName = entityViewById.getName(); + } + } + } + return entityName; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessorV1.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessorV1.java index 30ee4efe5f..18e65e6bac 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessorV1.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessorV1.java @@ -59,9 +59,9 @@ public class AlarmEdgeProcessorV1 extends AlarmEdgeProcessor { private EntityId getAlarmOriginator(TenantId tenantId, String entityName, EntityType entityType) { return switch (entityType) { - case DEVICE -> deviceService.findDeviceByTenantIdAndName(tenantId, entityName).getId(); - case ASSET -> assetService.findAssetByTenantIdAndName(tenantId, entityName).getId(); - case ENTITY_VIEW -> entityViewService.findEntityViewByTenantIdAndName(tenantId, entityName).getId(); + case DEVICE -> edgeCtx.getDeviceService().findDeviceByTenantIdAndName(tenantId, entityName).getId(); + case ASSET -> edgeCtx.getAssetService().findAssetByTenantIdAndName(tenantId, entityName).getId(); + case ENTITY_VIEW -> edgeCtx.getEntityViewService().findEntityViewByTenantIdAndName(tenantId, entityName).getId(); default -> null; }; } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/BaseAlarmProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/BaseAlarmProcessor.java index f6a05e363c..195b569ab4 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/BaseAlarmProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/BaseAlarmProcessor.java @@ -15,32 +15,21 @@ */ package org.thingsboard.server.service.edge.rpc.processor.alarm; -import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmComment; import org.thingsboard.server.common.data.alarm.AlarmCreateOrUpdateActiveRequest; import org.thingsboard.server.common.data.alarm.AlarmUpdateRequest; -import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.AlarmId; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.alarm.AlarmCommentDao; import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; -import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.service.edge.rpc.constructor.alarm.AlarmMsgConstructor; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; import java.util.UUID; @@ -66,27 +55,27 @@ public abstract class BaseAlarmProcessor extends BaseEdgeProcessor { try { switch (alarmUpdateMsg.getMsgType()) { case ENTITY_CREATED_RPC_MESSAGE: - alarmService.createAlarm(AlarmCreateOrUpdateActiveRequest.fromAlarm(alarm, null, alarmId)); + edgeCtx.getAlarmService().createAlarm(AlarmCreateOrUpdateActiveRequest.fromAlarm(alarm, null, alarmId)); break; case ENTITY_UPDATED_RPC_MESSAGE: - alarmService.updateAlarm(AlarmUpdateRequest.fromAlarm(alarm)); + edgeCtx.getAlarmService().updateAlarm(AlarmUpdateRequest.fromAlarm(alarm)); break; case ALARM_ACK_RPC_MESSAGE: - Alarm alarmToAck = alarmService.findAlarmById(tenantId, alarmId); + Alarm alarmToAck = edgeCtx.getAlarmService().findAlarmById(tenantId, alarmId); if (alarmToAck != null) { - alarmService.acknowledgeAlarm(tenantId, alarmId, alarm.getAckTs()); + edgeCtx.getAlarmService().acknowledgeAlarm(tenantId, alarmId, alarm.getAckTs()); } break; case ALARM_CLEAR_RPC_MESSAGE: - Alarm alarmToClear = alarmService.findAlarmById(tenantId, alarmId); + Alarm alarmToClear = edgeCtx.getAlarmService().findAlarmById(tenantId, alarmId); if (alarmToClear != null) { - alarmService.clearAlarm(tenantId, alarmId, alarm.getClearTs(), alarm.getDetails()); + edgeCtx.getAlarmService().clearAlarm(tenantId, alarmId, alarm.getClearTs(), alarm.getDetails()); } break; case ENTITY_DELETED_RPC_MESSAGE: - Alarm alarmToDelete = alarmService.findAlarmById(tenantId, alarmId); + Alarm alarmToDelete = edgeCtx.getAlarmService().findAlarmById(tenantId, alarmId); if (alarmToDelete != null) { - alarmService.delAlarm(tenantId, alarmId); + edgeCtx.getAlarmService().delAlarm(tenantId, alarmId); } break; case UNRECOGNIZED: @@ -107,7 +96,7 @@ public abstract class BaseAlarmProcessor extends BaseEdgeProcessor { throw new RuntimeException("[{" + tenantId + "}] alarmCommentUpdateMsg {" + alarmCommentUpdateMsg + "} cannot be converted to alarm comment"); } try { - Alarm alarm = alarmService.findAlarmById(tenantId, new AlarmId(alarmComment.getAlarmId().getId())); + Alarm alarm = edgeCtx.getAlarmService().findAlarmById(tenantId, new AlarmId(alarmComment.getAlarmId().getId())); if (alarm == null) { return Futures.immediateFuture(null); } @@ -116,12 +105,12 @@ public abstract class BaseAlarmProcessor extends BaseEdgeProcessor { alarmCommentDao.save(tenantId, alarmComment); break; case ENTITY_UPDATED_RPC_MESSAGE: - alarmCommentService.createOrUpdateAlarmComment(tenantId, alarmComment); + edgeCtx.getAlarmCommentService().createOrUpdateAlarmComment(tenantId, alarmComment); break; case ENTITY_DELETED_RPC_MESSAGE: - AlarmComment alarmCommentToDelete = alarmCommentService.findAlarmCommentById(tenantId, alarmComment.getId()); + AlarmComment alarmCommentToDelete = edgeCtx.getAlarmCommentService().findAlarmCommentById(tenantId, alarmComment.getId()); if (alarmCommentToDelete != null) { - alarmCommentService.saveAlarmComment(tenantId, alarmCommentToDelete); + edgeCtx.getAlarmCommentService().saveAlarmComment(tenantId, alarmCommentToDelete); } break; case UNRECOGNIZED: @@ -140,51 +129,4 @@ public abstract class BaseAlarmProcessor extends BaseEdgeProcessor { protected abstract Alarm constructAlarmFromUpdateMsg(TenantId tenantId, AlarmId alarmId, EntityId originatorId, AlarmUpdateMsg alarmUpdateMsg); - protected AlarmUpdateMsg convertAlarmEventToAlarmMsg(TenantId tenantId, UUID entityId, EdgeEventActionType actionType, JsonNode body, EdgeVersion edgeVersion) { - AlarmId alarmId = new AlarmId(entityId); - UpdateMsgType msgType = getUpdateMsgType(actionType); - switch (actionType) { - case ADDED, UPDATED, ALARM_ACK, ALARM_CLEAR -> { - Alarm alarm = alarmService.findAlarmById(tenantId, alarmId); - if (alarm != null) { - return ((AlarmMsgConstructor) alarmMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)) - .constructAlarmUpdatedMsg(msgType, alarm, findOriginatorEntityName(tenantId, alarm)); - } - } - case ALARM_DELETE, DELETED -> { - Alarm deletedAlarm = JacksonUtil.convertValue(body, Alarm.class); - if (deletedAlarm != null) { - return ((AlarmMsgConstructor) alarmMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)) - .constructAlarmUpdatedMsg(msgType, deletedAlarm, findOriginatorEntityName(tenantId, deletedAlarm)); - } - } - } - return null; - } - - private String findOriginatorEntityName(TenantId tenantId, Alarm alarm) { - String entityName = null; - switch (alarm.getOriginator().getEntityType()) { - case DEVICE -> { - Device deviceById = deviceService.findDeviceById(tenantId, new DeviceId(alarm.getOriginator().getId())); - if (deviceById != null) { - entityName = deviceById.getName(); - } - } - case ASSET -> { - Asset assetById = assetService.findAssetById(tenantId, new AssetId(alarm.getOriginator().getId())); - if (assetById != null) { - entityName = assetById.getName(); - } - } - case ENTITY_VIEW -> { - EntityView entityViewById = entityViewService.findEntityViewById(tenantId, new EntityViewId(alarm.getOriginator().getId())); - if (entityViewById != null) { - entityName = entityViewById.getName(); - } - } - } - return entityName; - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java index 3e3c9dfb64..56a4864763 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java @@ -28,7 +28,6 @@ import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.msg.TbMsgMetaData; @@ -57,9 +56,9 @@ public abstract class AssetEdgeProcessor extends BaseAssetProcessor implements A saveOrUpdateAsset(tenantId, assetId, assetUpdateMsg, edge); return Futures.immediateFuture(null); case ENTITY_DELETED_RPC_MESSAGE: - Asset assetToDelete = assetService.findAssetById(tenantId, assetId); + Asset assetToDelete = edgeCtx.getAssetService().findAssetById(tenantId, assetId); if (assetToDelete != null) { - assetService.unassignAssetFromEdge(tenantId, assetId, edge.getId()); + edgeCtx.getAssetService().unassignAssetFromEdge(tenantId, assetId, edge.getId()); } return Futures.immediateFuture(null); case UNRECOGNIZED: @@ -84,7 +83,7 @@ public abstract class AssetEdgeProcessor extends BaseAssetProcessor implements A if (created) { createRelationFromEdge(tenantId, edge.getId(), assetId); pushAssetCreatedEventToRuleEngine(tenantId, edge, assetId); - assetService.assignAssetToEdge(tenantId, assetId, edge.getId()); + edgeCtx.getAssetService().assignAssetToEdge(tenantId, assetId, edge.getId()); } Boolean assetNameUpdated = resultPair.getSecond(); if (assetNameUpdated) { @@ -94,7 +93,7 @@ public abstract class AssetEdgeProcessor extends BaseAssetProcessor implements A private void pushAssetCreatedEventToRuleEngine(TenantId tenantId, Edge edge, AssetId assetId) { try { - Asset asset = assetService.findAssetById(tenantId, assetId); + Asset asset = edgeCtx.getAssetService().findAssetById(tenantId, assetId); String assetAsString = JacksonUtil.toString(asset); TbMsgMetaData msgMetaData = getEdgeActionTbMsgMetaData(edge, asset.getCustomerId()); pushEntityEventToRuleEngine(tenantId, assetId, asset.getCustomerId(), TbMsgType.ENTITY_CREATED, assetAsString, msgMetaData); @@ -104,34 +103,33 @@ public abstract class AssetEdgeProcessor extends BaseAssetProcessor implements A } @Override - public DownlinkMsg convertAssetEventToDownlink(EdgeEvent edgeEvent, EdgeId edgeId, EdgeVersion edgeVersion) { + public DownlinkMsg convertAssetEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { AssetId assetId = new AssetId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; - var msgConstructor = (AssetMsgConstructor) assetMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); + var msgConstructor = (AssetMsgConstructor) edgeCtx.getAssetMsgConstructorFactory().getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED, ASSIGNED_TO_EDGE, ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER -> { - Asset asset = assetService.findAssetById(edgeEvent.getTenantId(), assetId); + Asset asset = edgeCtx.getAssetService().findAssetById(edgeEvent.getTenantId(), assetId); if (asset != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - AssetUpdateMsg assetUpdateMsg = ((AssetMsgConstructor) - assetMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructAssetUpdatedMsg(msgType, asset); + AssetUpdateMsg assetUpdateMsg = msgConstructor.constructAssetUpdatedMsg(msgType, asset); DownlinkMsg.Builder builder = DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addAssetUpdateMsg(assetUpdateMsg); if (UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE.equals(msgType)) { - AssetProfile assetProfile = assetProfileService.findAssetProfileById(edgeEvent.getTenantId(), asset.getAssetProfileId()); - assetProfile = checkIfAssetProfileDefaultFieldsAssignedToEdge(edgeEvent.getTenantId(), edgeId, assetProfile, edgeVersion); + AssetProfile assetProfile = edgeCtx.getAssetProfileService().findAssetProfileById(edgeEvent.getTenantId(), asset.getAssetProfileId()); builder.addAssetProfileUpdateMsg(msgConstructor.constructAssetProfileUpdatedMsg(msgType, assetProfile)); } - downlinkMsg = builder.build(); + return builder.build(); } } - case DELETED, UNASSIGNED_FROM_EDGE -> downlinkMsg = DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addAssetUpdateMsg(msgConstructor.constructAssetDeleteMsg(assetId)) - .build(); + case DELETED, UNASSIGNED_FROM_EDGE -> { + return DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addAssetUpdateMsg(msgConstructor.constructAssetDeleteMsg(assetId)) + .build(); + } } - return downlinkMsg; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetProcessor.java index 2685274bbf..f6b015701c 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetProcessor.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.edge.rpc.processor.asset; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; -import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; @@ -29,6 +28,6 @@ public interface AssetProcessor extends EdgeProcessor { ListenableFuture processAssetMsgFromEdge(TenantId tenantId, Edge edge, AssetUpdateMsg assetUpdateMsg); - DownlinkMsg convertAssetEventToDownlink(EdgeEvent edgeEvent, EdgeId edgeId, EdgeVersion edgeVersion); + DownlinkMsg convertAssetEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/BaseAssetProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/BaseAssetProcessor.java index 2588e52d5f..9ebcd74888 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/BaseAssetProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/BaseAssetProcessor.java @@ -16,18 +16,23 @@ package org.thingsboard.server.service.edge.rpc.processor.asset; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.util.Pair; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j public abstract class BaseAssetProcessor extends BaseEdgeProcessor { + @Autowired + private DataValidator assetValidator; + protected Pair saveOrUpdateAsset(TenantId tenantId, AssetId assetId, AssetUpdateMsg assetUpdateMsg) { boolean created = false; boolean assetNameUpdated = false; @@ -37,7 +42,7 @@ public abstract class BaseAssetProcessor extends BaseEdgeProcessor { if (asset == null) { throw new RuntimeException("[{" + tenantId + "}] assetUpdateMsg {" + assetUpdateMsg + " } cannot be converted to asset"); } - Asset assetById = assetService.findAssetById(tenantId, assetId); + Asset assetById = edgeCtx.getAssetService().findAssetById(tenantId, assetId); if (assetById == null) { created = true; asset.setId(null); @@ -45,7 +50,7 @@ public abstract class BaseAssetProcessor extends BaseEdgeProcessor { asset.setId(assetId); } String assetName = asset.getName(); - Asset assetByName = assetService.findAssetByTenantIdAndName(tenantId, assetName); + Asset assetByName = edgeCtx.getAssetService().findAssetByTenantIdAndName(tenantId, assetName); if (assetByName != null && !assetByName.getId().equals(assetId)) { assetName = assetName + "_" + StringUtils.randomAlphanumeric(15); log.warn("[{}] Asset with name {} already exists. Renaming asset name to {}", @@ -59,7 +64,7 @@ public abstract class BaseAssetProcessor extends BaseEdgeProcessor { if (created) { asset.setId(assetId); } - assetService.saveAsset(asset, false); + edgeCtx.getAssetService().saveAsset(asset, false); } catch (Exception e) { log.error("[{}] Failed to process asset update msg [{}]", tenantId, assetUpdateMsg, e); throw e; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessor.java index b4d56ed20f..1998def9fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessor.java @@ -50,16 +50,13 @@ public abstract class AssetProfileEdgeProcessor extends BaseAssetProfileProcesso try { edgeSynchronizationManager.getEdgeId().set(edge.getId()); - switch (assetProfileUpdateMsg.getMsgType()) { - case ENTITY_CREATED_RPC_MESSAGE: - case ENTITY_UPDATED_RPC_MESSAGE: + return switch (assetProfileUpdateMsg.getMsgType()) { + case ENTITY_CREATED_RPC_MESSAGE, ENTITY_UPDATED_RPC_MESSAGE -> { saveOrUpdateAssetProfile(tenantId, assetProfileId, assetProfileUpdateMsg, edge); - return Futures.immediateFuture(null); - case ENTITY_DELETED_RPC_MESSAGE: - case UNRECOGNIZED: - default: - return handleUnsupportedMsgType(assetProfileUpdateMsg.getMsgType()); - } + yield Futures.immediateFuture(null); + } + default -> handleUnsupportedMsgType(assetProfileUpdateMsg.getMsgType()); + }; } catch (DataValidationException e) { log.warn("[{}] Failed to process AssetProfileUpdateMsg from Edge [{}]", tenantId, assetProfileUpdateMsg, e); return Futures.immediateFailedFuture(e); @@ -83,7 +80,7 @@ public abstract class AssetProfileEdgeProcessor extends BaseAssetProfileProcesso private void pushAssetProfileCreatedEventToRuleEngine(TenantId tenantId, Edge edge, AssetProfileId assetProfileId) { try { - AssetProfile assetProfile = assetProfileService.findAssetProfileById(tenantId, assetProfileId); + AssetProfile assetProfile = edgeCtx.getAssetProfileService().findAssetProfileById(tenantId, assetProfileId); String assetProfileAsString = JacksonUtil.toString(assetProfile); TbMsgMetaData msgMetaData = getEdgeActionTbMsgMetaData(edge, null); pushEntityEventToRuleEngine(tenantId, assetProfileId, null, TbMsgType.ENTITY_CREATED, assetProfileAsString, msgMetaData); @@ -93,33 +90,30 @@ public abstract class AssetProfileEdgeProcessor extends BaseAssetProfileProcesso } @Override - public DownlinkMsg convertAssetProfileEventToDownlink(EdgeEvent edgeEvent, EdgeId edgeId, EdgeVersion edgeVersion) { + public DownlinkMsg convertAssetProfileEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { AssetProfileId assetProfileId = new AssetProfileId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; + var msgConstructor = (AssetMsgConstructor) edgeCtx.getAssetMsgConstructorFactory().getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { - AssetProfile assetProfile = assetProfileService.findAssetProfileById(edgeEvent.getTenantId(), assetProfileId); + AssetProfile assetProfile = edgeCtx.getAssetProfileService().findAssetProfileById(edgeEvent.getTenantId(), assetProfileId); if (assetProfile != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - assetProfile = checkIfAssetProfileDefaultFieldsAssignedToEdge(edgeEvent.getTenantId(), edgeId, assetProfile, edgeVersion); - AssetProfileUpdateMsg assetProfileUpdateMsg = ((AssetMsgConstructor) - assetMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructAssetProfileUpdatedMsg(msgType, assetProfile); - downlinkMsg = DownlinkMsg.newBuilder() + AssetProfileUpdateMsg assetProfileUpdateMsg = msgConstructor.constructAssetProfileUpdatedMsg(msgType, assetProfile); + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addAssetProfileUpdateMsg(assetProfileUpdateMsg) .build(); } } case DELETED -> { - AssetProfileUpdateMsg assetProfileUpdateMsg = ((AssetMsgConstructor) - assetMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructAssetProfileDeleteMsg(assetProfileId); - downlinkMsg = DownlinkMsg.newBuilder() + AssetProfileUpdateMsg assetProfileUpdateMsg = msgConstructor.constructAssetProfileDeleteMsg(assetProfileId); + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addAssetProfileUpdateMsg(assetProfileUpdateMsg) .build(); } } - return downlinkMsg; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileProcessor.java index ba7aa106cf..83051c519c 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileProcessor.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.edge.rpc.processor.asset.profile; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; -import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; @@ -29,5 +28,6 @@ public interface AssetProfileProcessor extends EdgeProcessor { ListenableFuture processAssetProfileMsgFromEdge(TenantId tenantId, Edge edge, AssetProfileUpdateMsg assetProfileUpdateMsg); - DownlinkMsg convertAssetProfileEventToDownlink(EdgeEvent edgeEvent, EdgeId edgeId, EdgeVersion edgeVersion); + DownlinkMsg convertAssetProfileEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion); + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/BaseAssetProfileProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/BaseAssetProfileProcessor.java index 3a4dbc86ed..2b1440a480 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/BaseAssetProfileProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/BaseAssetProfileProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.edge.rpc.processor.asset.profile; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.util.Pair; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.AssetProfile; @@ -23,12 +24,16 @@ import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j public abstract class BaseAssetProfileProcessor extends BaseEdgeProcessor { + @Autowired + private DataValidator assetProfileValidator; + protected Pair saveOrUpdateAssetProfile(TenantId tenantId, AssetProfileId assetProfileId, AssetProfileUpdateMsg assetProfileUpdateMsg) { boolean created = false; boolean assetProfileNameUpdated = false; @@ -38,7 +43,7 @@ public abstract class BaseAssetProfileProcessor extends BaseEdgeProcessor { if (assetProfile == null) { throw new RuntimeException("[{" + tenantId + "}] assetProfileUpdateMsg {" + assetProfileUpdateMsg + "} cannot be converted to asset profile"); } - AssetProfile assetProfileById = assetProfileService.findAssetProfileById(tenantId, assetProfileId); + AssetProfile assetProfileById = edgeCtx.getAssetProfileService().findAssetProfileById(tenantId, assetProfileId); if (assetProfileById == null) { created = true; assetProfile.setId(null); @@ -48,7 +53,7 @@ public abstract class BaseAssetProfileProcessor extends BaseEdgeProcessor { assetProfile.setDefault(assetProfileById.isDefault()); } String assetProfileName = assetProfile.getName(); - AssetProfile assetProfileByName = assetProfileService.findAssetProfileByName(tenantId, assetProfileName); + AssetProfile assetProfileByName = edgeCtx.getAssetProfileService().findAssetProfileByName(tenantId, assetProfileName); if (assetProfileByName != null && !assetProfileByName.getId().equals(assetProfileId)) { assetProfileName = assetProfileName + "_" + StringUtils.randomAlphabetic(15); log.warn("[{}] Asset profile with name {} already exists. Renaming asset profile name to {}", @@ -66,7 +71,7 @@ public abstract class BaseAssetProfileProcessor extends BaseEdgeProcessor { if (created) { assetProfile.setId(assetProfileId); } - assetProfileService.saveAssetProfile(assetProfile, false, true); + edgeCtx.getAssetProfileService().saveAssetProfile(assetProfile, false, true); } catch (Exception e) { log.error("[{}] Failed to process asset profile update msg [{}]", tenantId, assetProfileUpdateMsg, e); throw e; @@ -83,4 +88,5 @@ public abstract class BaseAssetProfileProcessor extends BaseEdgeProcessor { protected abstract void setDefaultEdgeRuleChainId(AssetProfile assetProfile, RuleChainId ruleChainId, AssetProfileUpdateMsg assetProfileUpdateMsg); protected abstract void setDefaultDashboardId(TenantId tenantId, DashboardId dashboardId, AssetProfile assetProfile, AssetProfileUpdateMsg assetProfileUpdateMsg); + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/customer/CustomerEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/customer/CustomerEdgeProcessor.java index e13c4c487f..ddf6f25728 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/customer/CustomerEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/customer/CustomerEdgeProcessor.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.edge.rpc.processor.customer; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EdgeUtils; @@ -37,6 +38,7 @@ import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.rpc.constructor.customer.CustomerMsgConstructor; +import org.thingsboard.server.service.edge.rpc.constructor.customer.CustomerMsgConstructorFactory; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; import java.util.ArrayList; @@ -48,32 +50,33 @@ import java.util.UUID; @TbCoreComponent public class CustomerEdgeProcessor extends BaseEdgeProcessor { + @Autowired + private CustomerMsgConstructorFactory customerMsgConstructorFactory; + public DownlinkMsg convertCustomerEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { CustomerId customerId = new CustomerId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; + var msgConstructor = (CustomerMsgConstructor) customerMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { - Customer customer = customerService.findCustomerById(edgeEvent.getTenantId(), customerId); + Customer customer = edgeCtx.getCustomerService().findCustomerById(edgeEvent.getTenantId(), customerId); if (customer != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - CustomerUpdateMsg customerUpdateMsg = ((CustomerMsgConstructor) - customerMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructCustomerUpdatedMsg(msgType, customer); - downlinkMsg = DownlinkMsg.newBuilder() + CustomerUpdateMsg customerUpdateMsg = msgConstructor.constructCustomerUpdatedMsg(msgType, customer); + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addCustomerUpdateMsg(customerUpdateMsg) .build(); } } case DELETED -> { - CustomerUpdateMsg customerUpdateMsg = ((CustomerMsgConstructor) - customerMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructCustomerDeleteMsg(customerId); - downlinkMsg = DownlinkMsg.newBuilder() + CustomerUpdateMsg customerUpdateMsg = msgConstructor.constructCustomerDeleteMsg(customerId); + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addCustomerUpdateMsg(customerUpdateMsg) .build(); } } - return downlinkMsg; + return null; } public ListenableFuture processCustomerNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) { @@ -84,7 +87,7 @@ public class CustomerEdgeProcessor extends BaseEdgeProcessor { switch (actionType) { case UPDATED: List> futures = new ArrayList<>(); - PageDataIterable edges = new PageDataIterable<>(link -> edgeService.findEdgesByTenantIdAndCustomerId(tenantId, customerId, link), 1024); + PageDataIterable edges = new PageDataIterable<>(link -> edgeCtx.getEdgeService().findEdgesByTenantIdAndCustomerId(tenantId, customerId, link), 1024); for (Edge edge : edges) { futures.add(saveEdgeEvent(tenantId, edge.getId(), type, actionType, customerId, null)); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java index 64fb1e3981..52277b8527 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java @@ -16,11 +16,13 @@ package org.thingsboard.server.service.edge.rpc.processor.dashboard; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.ShortCustomerInfo; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @@ -29,6 +31,9 @@ import java.util.Set; @Slf4j public abstract class BaseDashboardProcessor extends BaseEdgeProcessor { + @Autowired + private DataValidator dashboardValidator; + protected boolean saveOrUpdateDashboard(TenantId tenantId, DashboardId dashboardId, DashboardUpdateMsg dashboardUpdateMsg, CustomerId customerId) { boolean created = false; Dashboard dashboard = constructDashboardFromUpdateMsg(tenantId, dashboardId, dashboardUpdateMsg); @@ -36,7 +41,7 @@ public abstract class BaseDashboardProcessor extends BaseEdgeProcessor { throw new RuntimeException("[{" + tenantId + "}] dashboardUpdateMsg {" + dashboardUpdateMsg + "} cannot be converted to dashboard"); } Set assignedCustomers = null; - Dashboard dashboardById = dashboardService.findDashboardById(tenantId, dashboardId); + Dashboard dashboardById = edgeCtx.getDashboardService().findDashboardById(tenantId, dashboardId); if (dashboardById == null) { created = true; dashboard.setId(null); @@ -58,11 +63,11 @@ public abstract class BaseDashboardProcessor extends BaseEdgeProcessor { } } dashboard.setAssignedCustomers(assignedCustomers); - Dashboard savedDashboard = dashboardService.saveDashboard(dashboard, false); + Dashboard savedDashboard = edgeCtx.getDashboardService().saveDashboard(dashboard, false); if (msgAssignedCustomers != null && !msgAssignedCustomers.isEmpty()) { for (ShortCustomerInfo assignedCustomer : msgAssignedCustomers) { if (assignedCustomer.getCustomerId().equals(customerId)) { - dashboardService.assignDashboardToCustomer(tenantId, savedDashboard.getId(), assignedCustomer.getCustomerId()); + edgeCtx.getDashboardService().assignDashboardToCustomer(tenantId, savedDashboard.getId(), assignedCustomer.getCustomerId()); } } } else { @@ -75,7 +80,7 @@ public abstract class BaseDashboardProcessor extends BaseEdgeProcessor { if (dashboard.getAssignedCustomers() != null && !dashboard.getAssignedCustomers().isEmpty()) { for (ShortCustomerInfo assignedCustomer : dashboard.getAssignedCustomers()) { if (assignedCustomer.getCustomerId().equals(customerId)) { - dashboardService.unassignDashboardFromCustomer(tenantId, dashboard.getId(), assignedCustomer.getCustomerId()); + edgeCtx.getDashboardService().unassignDashboardFromCustomer(tenantId, dashboard.getId(), assignedCustomer.getCustomerId()); } } } @@ -84,4 +89,5 @@ public abstract class BaseDashboardProcessor extends BaseEdgeProcessor { protected abstract Dashboard constructDashboardFromUpdateMsg(TenantId tenantId, DashboardId dashboardId, DashboardUpdateMsg dashboardUpdateMsg); protected abstract Set filterNonExistingCustomers(TenantId tenantId, Set assignedCustomers); + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java index f5f25b1c56..cb54977c3c 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.edge.rpc.processor.dashboard; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EdgeUtils; @@ -34,6 +35,7 @@ import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.service.edge.rpc.constructor.dashboard.DashboardMsgConstructor; +import org.thingsboard.server.service.edge.rpc.constructor.dashboard.DashboardMsgConstructorFactory; import java.util.Set; import java.util.UUID; @@ -41,6 +43,9 @@ import java.util.UUID; @Slf4j public abstract class DashboardEdgeProcessor extends BaseDashboardProcessor implements DashboardProcessor { + @Autowired + private DashboardMsgConstructorFactory dashboardMsgConstructorFactory; + @Override public ListenableFuture processDashboardMsgFromEdge(TenantId tenantId, Edge edge, DashboardUpdateMsg dashboardUpdateMsg) { log.trace("[{}] executing processDashboardMsgFromEdge [{}] from edge [{}]", tenantId, dashboardUpdateMsg, edge.getId()); @@ -54,9 +59,9 @@ public abstract class DashboardEdgeProcessor extends BaseDashboardProcessor impl saveOrUpdateDashboard(tenantId, dashboardId, dashboardUpdateMsg, edge); return Futures.immediateFuture(null); case ENTITY_DELETED_RPC_MESSAGE: - Dashboard dashboardToDelete = dashboardService.findDashboardById(tenantId, dashboardId); + Dashboard dashboardToDelete = edgeCtx.getDashboardService().findDashboardById(tenantId, dashboardId); if (dashboardToDelete != null) { - dashboardService.unassignDashboardFromEdge(tenantId, dashboardId, edge.getId()); + edgeCtx.getDashboardService().unassignDashboardFromEdge(tenantId, dashboardId, edge.getId()); } return Futures.immediateFuture(null); case UNRECOGNIZED: @@ -76,18 +81,17 @@ public abstract class DashboardEdgeProcessor extends BaseDashboardProcessor impl } private void saveOrUpdateDashboard(TenantId tenantId, DashboardId dashboardId, DashboardUpdateMsg dashboardUpdateMsg, Edge edge) { - boolean created = super.saveOrUpdateDashboard(tenantId, dashboardId, dashboardUpdateMsg, - edge.getCustomerId()); + boolean created = super.saveOrUpdateDashboard(tenantId, dashboardId, dashboardUpdateMsg, edge.getCustomerId()); if (created) { createRelationFromEdge(tenantId, edge.getId(), dashboardId); pushDashboardCreatedEventToRuleEngine(tenantId, edge, dashboardId); - dashboardService.assignDashboardToEdge(tenantId, dashboardId, edge.getId()); + edgeCtx.getDashboardService().assignDashboardToEdge(tenantId, dashboardId, edge.getId()); } } private void pushDashboardCreatedEventToRuleEngine(TenantId tenantId, Edge edge, DashboardId dashboardId) { try { - Dashboard dashboard = dashboardService.findDashboardById(tenantId, dashboardId); + Dashboard dashboard = edgeCtx.getDashboardService().findDashboardById(tenantId, dashboardId); String dashboardAsString = JacksonUtil.toString(dashboard); TbMsgMetaData msgMetaData = getEdgeActionTbMsgMetaData(edge, null); pushEntityEventToRuleEngine(tenantId, dashboardId, null, TbMsgType.ENTITY_CREATED, dashboardAsString, msgMetaData); @@ -99,26 +103,28 @@ public abstract class DashboardEdgeProcessor extends BaseDashboardProcessor impl @Override public DownlinkMsg convertDashboardEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { DashboardId dashboardId = new DashboardId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; var msgConstructor = (DashboardMsgConstructor) dashboardMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED, ASSIGNED_TO_EDGE, ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER -> { - Dashboard dashboard = dashboardService.findDashboardById(edgeEvent.getTenantId(), dashboardId); + Dashboard dashboard = edgeCtx.getDashboardService().findDashboardById(edgeEvent.getTenantId(), dashboardId); if (dashboard != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); DashboardUpdateMsg dashboardUpdateMsg = msgConstructor.constructDashboardUpdatedMsg(msgType, dashboard); - downlinkMsg = DownlinkMsg.newBuilder() + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addDashboardUpdateMsg(dashboardUpdateMsg) .build(); } } - case DELETED, UNASSIGNED_FROM_EDGE -> downlinkMsg = DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addDashboardUpdateMsg(msgConstructor.constructDashboardDeleteMsg(dashboardId)) - .build(); + case DELETED, UNASSIGNED_FROM_EDGE -> { + DashboardUpdateMsg dashboardUpdateMsg = msgConstructor.constructDashboardDeleteMsg(dashboardId); + return DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addDashboardUpdateMsg(dashboardUpdateMsg) + .build(); + } } - return downlinkMsg; + return null; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorV2.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorV2.java index 36369edde3..10bfe555a6 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorV2.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessorV2.java @@ -33,4 +33,5 @@ public class DashboardEdgeProcessorV2 extends DashboardEdgeProcessor { protected Dashboard constructDashboardFromUpdateMsg(TenantId tenantId, DashboardId dashboardId, DashboardUpdateMsg dashboardUpdateMsg) { return JacksonUtil.fromString(dashboardUpdateMsg.getEntity(), Dashboard.class, true); } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/BaseDeviceProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/BaseDeviceProcessor.java index 016273bef9..4b2757e510 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/BaseDeviceProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/BaseDeviceProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.edge.rpc.processor.device; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.util.Pair; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.StringUtils; @@ -23,6 +24,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @@ -30,6 +32,9 @@ import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j public abstract class BaseDeviceProcessor extends BaseEdgeProcessor { + @Autowired + private DataValidator deviceValidator; + protected Pair saveOrUpdateDevice(TenantId tenantId, DeviceId deviceId, DeviceUpdateMsg deviceUpdateMsg) { boolean created = false; boolean deviceNameUpdated = false; @@ -39,7 +44,7 @@ public abstract class BaseDeviceProcessor extends BaseEdgeProcessor { if (device == null) { throw new RuntimeException("[{" + tenantId + "}] deviceUpdateMsg {" + deviceUpdateMsg + "} cannot be converted to device"); } - Device deviceById = deviceService.findDeviceById(tenantId, deviceId); + Device deviceById = edgeCtx.getDeviceService().findDeviceById(tenantId, deviceId); if (deviceById == null) { created = true; device.setId(null); @@ -47,7 +52,7 @@ public abstract class BaseDeviceProcessor extends BaseEdgeProcessor { device.setId(deviceId); } String deviceName = device.getName(); - Device deviceByName = deviceService.findDeviceByTenantIdAndName(tenantId, deviceName); + Device deviceByName = edgeCtx.getDeviceService().findDeviceByTenantIdAndName(tenantId, deviceName); if (deviceByName != null && !deviceByName.getId().equals(deviceId)) { deviceName = deviceName + "_" + StringUtils.randomAlphabetic(15); log.warn("[{}] Device with name {} already exists. Renaming device name to {}", @@ -61,8 +66,8 @@ public abstract class BaseDeviceProcessor extends BaseEdgeProcessor { if (created) { device.setId(deviceId); } - Device savedDevice = deviceService.saveDevice(device, false); - tbClusterService.onDeviceUpdated(savedDevice, created ? null : device); + Device savedDevice = edgeCtx.getDeviceService().saveDevice(device, false); + edgeCtx.getClusterService().onDeviceUpdated(savedDevice, created ? null : device); } catch (Exception e) { log.error("[{}] Failed to process device update msg [{}]", tenantId, deviceUpdateMsg, e); throw e; @@ -77,12 +82,12 @@ public abstract class BaseDeviceProcessor extends BaseEdgeProcessor { if (deviceCredentials == null) { throw new RuntimeException("[{" + tenantId + "}] deviceCredentialsUpdateMsg {" + deviceCredentialsUpdateMsg + "} cannot be converted to device credentials"); } - Device device = deviceService.findDeviceById(tenantId, deviceCredentials.getDeviceId()); + Device device = edgeCtx.getDeviceService().findDeviceById(tenantId, deviceCredentials.getDeviceId()); if (device != null) { log.debug("[{}] Updating device credentials for device [{}]. New device credentials Id [{}], value [{}]", tenantId, device.getName(), deviceCredentials.getCredentialsId(), deviceCredentials.getCredentialsValue()); try { - DeviceCredentials deviceCredentialsByDeviceId = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, device.getId()); + DeviceCredentials deviceCredentialsByDeviceId = edgeCtx.getDeviceCredentialsService().findDeviceCredentialsByDeviceId(tenantId, device.getId()); if (deviceCredentialsByDeviceId == null) { deviceCredentialsByDeviceId = new DeviceCredentials(); deviceCredentialsByDeviceId.setDeviceId(device.getId()); @@ -90,7 +95,7 @@ public abstract class BaseDeviceProcessor extends BaseEdgeProcessor { deviceCredentialsByDeviceId.setCredentialsType(deviceCredentials.getCredentialsType()); deviceCredentialsByDeviceId.setCredentialsId(deviceCredentials.getCredentialsId()); deviceCredentialsByDeviceId.setCredentialsValue(deviceCredentials.getCredentialsValue()); - deviceCredentialsService.updateDeviceCredentials(tenantId, deviceCredentialsByDeviceId); + edgeCtx.getDeviceCredentialsService().updateDeviceCredentials(tenantId, deviceCredentialsByDeviceId); } catch (Exception e) { log.error("[{}] Can't update device credentials for device [{}], deviceCredentialsUpdateMsg [{}]", diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java index 52f09971ab..e453cbc26f 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java @@ -71,9 +71,9 @@ public abstract class DeviceEdgeProcessor extends BaseDeviceProcessor implements saveOrUpdateDevice(tenantId, deviceId, deviceUpdateMsg, edge); return Futures.immediateFuture(null); case ENTITY_DELETED_RPC_MESSAGE: - Device deviceToDelete = deviceService.findDeviceById(tenantId, deviceId); + Device deviceToDelete = edgeCtx.getDeviceService().findDeviceById(tenantId, deviceId); if (deviceToDelete != null) { - deviceService.unassignDeviceFromEdge(tenantId, deviceId, edge.getId()); + edgeCtx.getDeviceService().unassignDeviceFromEdge(tenantId, deviceId, edge.getId()); } return Futures.immediateFuture(null); case UNRECOGNIZED: @@ -111,7 +111,7 @@ public abstract class DeviceEdgeProcessor extends BaseDeviceProcessor implements if (created) { createRelationFromEdge(tenantId, edge.getId(), deviceId); pushDeviceCreatedEventToRuleEngine(tenantId, edge, deviceId); - deviceService.assignDeviceToEdge(tenantId, deviceId, edge.getId()); + edgeCtx.getDeviceService().assignDeviceToEdge(tenantId, deviceId, edge.getId()); } Boolean deviceNameUpdated = resultPair.getSecond(); if (deviceNameUpdated) { @@ -121,7 +121,7 @@ public abstract class DeviceEdgeProcessor extends BaseDeviceProcessor implements private void pushDeviceCreatedEventToRuleEngine(TenantId tenantId, Edge edge, DeviceId deviceId) { try { - Device device = deviceService.findDeviceById(tenantId, deviceId); + Device device = edgeCtx.getDeviceService().findDeviceById(tenantId, deviceId); String deviceAsString = JacksonUtil.toString(device); TbMsgMetaData msgMetaData = getEdgeActionTbMsgMetaData(edge, device.getCustomerId()); pushEntityEventToRuleEngine(tenantId, deviceId, device.getCustomerId(), TbMsgType.ENTITY_CREATED, deviceAsString, msgMetaData); @@ -168,7 +168,7 @@ public abstract class DeviceEdgeProcessor extends BaseDeviceProcessor implements new FromDeviceRpcResponseActorMsg(deviceRpcCallMsg.getRequestId(), tenantId, deviceId, response); - tbClusterService.pushMsgToCore(msg, callback); + edgeCtx.getClusterService().pushMsgToCore(msg, callback); return futureToSet; } @@ -181,7 +181,7 @@ public abstract class DeviceEdgeProcessor extends BaseDeviceProcessor implements metaData.putValue("serviceId", deviceRpcCallMsg.getServiceId()); metaData.putValue("sessionId", deviceRpcCallMsg.getSessionId()); metaData.putValue(DataConstants.EDGE_ID, edge.getId().toString()); - Device device = deviceService.findDeviceById(tenantId, deviceId); + Device device = edgeCtx.getDeviceService().findDeviceById(tenantId, deviceId); if (device != null) { metaData.putValue("deviceName", device.getName()); metaData.putValue("deviceType", device.getType()); @@ -192,7 +192,7 @@ public abstract class DeviceEdgeProcessor extends BaseDeviceProcessor implements data.put("params", deviceRpcCallMsg.getRequestMsg().getParams()); TbMsg tbMsg = TbMsg.newMsg(TbMsgType.TO_SERVER_RPC_REQUEST, deviceId, null, metaData, TbMsgDataType.JSON, JacksonUtil.toString(data)); - tbClusterService.pushMsgToRuleEngine(tenantId, deviceId, tbMsg, new TbQueueCallback() { + edgeCtx.getClusterService().pushMsgToRuleEngine(tenantId, deviceId, tbMsg, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { log.debug("[{}] Successfully send TO_SERVER_RPC_REQUEST to rule engine [{}], deviceRpcCallMsg {}", @@ -213,47 +213,44 @@ public abstract class DeviceEdgeProcessor extends BaseDeviceProcessor implements } @Override - public DownlinkMsg convertDeviceEventToDownlink(EdgeEvent edgeEvent, EdgeId edgeId, EdgeVersion edgeVersion) { + public DownlinkMsg convertDeviceEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { DeviceId deviceId = new DeviceId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; - var msgConstructor = (DeviceMsgConstructor) deviceMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); + var msgConstructor = (DeviceMsgConstructor) edgeCtx.getDeviceMsgConstructorFactory().getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED: case UPDATED: case ASSIGNED_TO_EDGE: case ASSIGNED_TO_CUSTOMER: case UNASSIGNED_FROM_CUSTOMER: - Device device = deviceService.findDeviceById(edgeEvent.getTenantId(), deviceId); + Device device = edgeCtx.getDeviceService().findDeviceById(edgeEvent.getTenantId(), deviceId); if (device != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); DeviceUpdateMsg deviceUpdateMsg = msgConstructor.constructDeviceUpdatedMsg(msgType, device); DownlinkMsg.Builder builder = DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addDeviceUpdateMsg(deviceUpdateMsg); - DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(edgeEvent.getTenantId(), deviceId); + DeviceCredentials deviceCredentials = edgeCtx.getDeviceCredentialsService().findDeviceCredentialsByDeviceId(edgeEvent.getTenantId(), deviceId); if (deviceCredentials != null) { DeviceCredentialsUpdateMsg deviceCredentialsUpdateMsg = msgConstructor.constructDeviceCredentialsUpdatedMsg(deviceCredentials); builder.addDeviceCredentialsUpdateMsg(deviceCredentialsUpdateMsg).build(); } if (UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE.equals(msgType)) { - DeviceProfile deviceProfile = deviceProfileService.findDeviceProfileById(edgeEvent.getTenantId(), device.getDeviceProfileId()); - deviceProfile = checkIfDeviceProfileDefaultFieldsAssignedToEdge(edgeEvent.getTenantId(), edgeId, deviceProfile, edgeVersion); + DeviceProfile deviceProfile = edgeCtx.getDeviceProfileService().findDeviceProfileById(edgeEvent.getTenantId(), device.getDeviceProfileId()); builder.addDeviceProfileUpdateMsg(msgConstructor.constructDeviceProfileUpdatedMsg(msgType, deviceProfile)); } - downlinkMsg = builder.build(); + return builder.build(); } break; case DELETED: case UNASSIGNED_FROM_EDGE: - downlinkMsg = DownlinkMsg.newBuilder() + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addDeviceUpdateMsg(msgConstructor.constructDeviceDeleteMsg(deviceId)) .build(); - break; case CREDENTIALS_UPDATED: - DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(edgeEvent.getTenantId(), deviceId); + DeviceCredentials deviceCredentials = edgeCtx.getDeviceCredentialsService().findDeviceCredentialsByDeviceId(edgeEvent.getTenantId(), deviceId); if (deviceCredentials != null) { - downlinkMsg = DownlinkMsg.newBuilder() + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addDeviceCredentialsUpdateMsg(msgConstructor.constructDeviceCredentialsUpdatedMsg(deviceCredentials)) .build(); @@ -265,7 +262,7 @@ public abstract class DeviceEdgeProcessor extends BaseDeviceProcessor implements .addDeviceRpcCallMsg(msgConstructor.constructDeviceRpcCallMsg(edgeEvent.getEntityId(), edgeEvent.getBody())) .build(); } - return downlinkMsg; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceProcessor.java index b55e47b294..087b5f36a7 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceProcessor.java @@ -33,7 +33,7 @@ public interface DeviceProcessor extends EdgeProcessor { ListenableFuture processDeviceCredentialsMsgFromEdge(TenantId tenantId, EdgeId edgeId, DeviceCredentialsUpdateMsg deviceCredentialsUpdateMsg); - DownlinkMsg convertDeviceEventToDownlink(EdgeEvent edgeEvent, EdgeId edgeId, EdgeVersion edgeVersion); + DownlinkMsg convertDeviceEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion); ListenableFuture processDeviceRpcCallFromEdge(TenantId tenantId, Edge edge, DeviceRpcCallMsg deviceRpcCallMsg); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/BaseDeviceProfileProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/BaseDeviceProfileProcessor.java index 5f84e40dee..d06618772d 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/BaseDeviceProfileProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/BaseDeviceProfileProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.edge.rpc.processor.device.profile; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.util.Pair; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.StringUtils; @@ -23,12 +24,16 @@ import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j public abstract class BaseDeviceProfileProcessor extends BaseEdgeProcessor { + @Autowired + private DataValidator deviceProfileValidator; + protected Pair saveOrUpdateDeviceProfile(TenantId tenantId, DeviceProfileId deviceProfileId, DeviceProfileUpdateMsg deviceProfileUpdateMsg) { boolean created = false; boolean deviceProfileNameUpdated = false; @@ -38,7 +43,7 @@ public abstract class BaseDeviceProfileProcessor extends BaseEdgeProcessor { if (deviceProfile == null) { throw new RuntimeException("[{" + tenantId + "}] deviceProfileUpdateMsg {" + deviceProfileUpdateMsg + "} cannot be converted to device profile"); } - DeviceProfile deviceProfileById = deviceProfileService.findDeviceProfileById(tenantId, deviceProfileId); + DeviceProfile deviceProfileById = edgeCtx.getDeviceProfileService().findDeviceProfileById(tenantId, deviceProfileId); if (deviceProfileById == null) { created = true; deviceProfile.setId(null); @@ -48,7 +53,7 @@ public abstract class BaseDeviceProfileProcessor extends BaseEdgeProcessor { deviceProfile.setDefault(deviceProfileById.isDefault()); } String deviceProfileName = deviceProfile.getName(); - DeviceProfile deviceProfileByName = deviceProfileService.findDeviceProfileByName(tenantId, deviceProfileName); + DeviceProfile deviceProfileByName = edgeCtx.getDeviceProfileService().findDeviceProfileByName(tenantId, deviceProfileName); if (deviceProfileByName != null && !deviceProfileByName.getId().equals(deviceProfileId)) { deviceProfileName = deviceProfileName + "_" + StringUtils.randomAlphabetic(15); log.warn("[{}] Device profile with name {} already exists. Renaming device profile name to {}", @@ -66,7 +71,7 @@ public abstract class BaseDeviceProfileProcessor extends BaseEdgeProcessor { if (created) { deviceProfile.setId(deviceProfileId); } - deviceProfileService.saveDeviceProfile(deviceProfile, false, true); + edgeCtx.getDeviceProfileService().saveDeviceProfile(deviceProfile, false, true); } catch (Exception e) { log.error("[{}] Failed to process device profile update msg [{}]", tenantId, deviceProfileUpdateMsg, e); throw e; @@ -83,4 +88,5 @@ public abstract class BaseDeviceProfileProcessor extends BaseEdgeProcessor { protected abstract void setDefaultEdgeRuleChainId(DeviceProfile deviceProfile, RuleChainId ruleChainId, DeviceProfileUpdateMsg deviceProfileUpdateMsg); protected abstract void setDefaultDashboardId(TenantId tenantId, DashboardId dashboardId, DeviceProfile deviceProfile, DeviceProfileUpdateMsg deviceProfileUpdateMsg); + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessor.java index f547cbc3a1..a0a699d340 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessor.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.msg.TbMsgMetaData; @@ -83,7 +82,7 @@ public abstract class DeviceProfileEdgeProcessor extends BaseDeviceProfileProces private void pushDeviceProfileCreatedEventToRuleEngine(TenantId tenantId, Edge edge, DeviceProfileId deviceProfileId) { try { - DeviceProfile deviceProfile = deviceProfileService.findDeviceProfileById(tenantId, deviceProfileId); + DeviceProfile deviceProfile = edgeCtx.getDeviceProfileService().findDeviceProfileById(tenantId, deviceProfileId); String deviceProfileAsString = JacksonUtil.toString(deviceProfile); TbMsgMetaData msgMetaData = getEdgeActionTbMsgMetaData(edge, null); pushEntityEventToRuleEngine(tenantId, deviceProfileId, null, TbMsgType.ENTITY_CREATED, deviceProfileAsString, msgMetaData); @@ -93,18 +92,16 @@ public abstract class DeviceProfileEdgeProcessor extends BaseDeviceProfileProces } @Override - public DownlinkMsg convertDeviceProfileEventToDownlink(EdgeEvent edgeEvent, EdgeId edgeId, EdgeVersion edgeVersion) { + public DownlinkMsg convertDeviceProfileEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { DeviceProfileId deviceProfileId = new DeviceProfileId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; - var msgConstructor = (DeviceMsgConstructor) deviceMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); + var msgConstructor = (DeviceMsgConstructor) edgeCtx.getDeviceMsgConstructorFactory().getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { - DeviceProfile deviceProfile = deviceProfileService.findDeviceProfileById(edgeEvent.getTenantId(), deviceProfileId); + DeviceProfile deviceProfile = edgeCtx.getDeviceProfileService().findDeviceProfileById(edgeEvent.getTenantId(), deviceProfileId); if (deviceProfile != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - deviceProfile = checkIfDeviceProfileDefaultFieldsAssignedToEdge(edgeEvent.getTenantId(), edgeId, deviceProfile, edgeVersion); DeviceProfileUpdateMsg deviceProfileUpdateMsg = msgConstructor.constructDeviceProfileUpdatedMsg(msgType, deviceProfile); - downlinkMsg = DownlinkMsg.newBuilder() + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addDeviceProfileUpdateMsg(deviceProfileUpdateMsg) .build(); @@ -112,13 +109,13 @@ public abstract class DeviceProfileEdgeProcessor extends BaseDeviceProfileProces } case DELETED -> { DeviceProfileUpdateMsg deviceProfileUpdateMsg = msgConstructor.constructDeviceProfileDeleteMsg(deviceProfileId); - downlinkMsg = DownlinkMsg.newBuilder() + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addDeviceProfileUpdateMsg(deviceProfileUpdateMsg) .build(); } } - return downlinkMsg; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileProcessor.java index 3f5fe6a8ed..817d35879d 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileProcessor.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.edge.rpc.processor.device.profile; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; -import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; @@ -29,5 +28,6 @@ public interface DeviceProfileProcessor extends EdgeProcessor { ListenableFuture processDeviceProfileMsgFromEdge(TenantId tenantId, Edge edge, DeviceProfileUpdateMsg deviceProfileUpdateMsg); - DownlinkMsg convertDeviceProfileEventToDownlink(EdgeEvent edgeEvent, EdgeId edgeId, EdgeVersion edgeVersion); + DownlinkMsg convertDeviceProfileEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion); + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeProcessor.java index f49c4b0a00..024147507a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeProcessor.java @@ -48,21 +48,19 @@ public class EdgeProcessor extends BaseEdgeProcessor { public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { EdgeId edgeId = new EdgeId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; switch (edgeEvent.getAction()) { case ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER -> { - Edge edge = edgeService.findEdgeById(edgeEvent.getTenantId(), edgeId); + Edge edge = edgeCtx.getEdgeService().findEdgeById(edgeEvent.getTenantId(), edgeId); if (edge != null) { - EdgeConfiguration edgeConfigMsg = - edgeMsgConstructor.constructEdgeConfiguration(edge); - downlinkMsg = DownlinkMsg.newBuilder() + EdgeConfiguration edgeConfigMsg = edgeCtx.getEdgeMsgConstructor().constructEdgeConfiguration(edge); + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .setEdgeConfiguration(edgeConfigMsg) .build(); } } } - return downlinkMsg; + return null; } public ListenableFuture processEdgeNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) { @@ -70,9 +68,9 @@ public class EdgeProcessor extends BaseEdgeProcessor { EdgeEventActionType actionType = EdgeEventActionType.valueOf(edgeNotificationMsg.getAction()); EdgeId edgeId = new EdgeId(new UUID(edgeNotificationMsg.getEntityIdMSB(), edgeNotificationMsg.getEntityIdLSB())); switch (actionType) { - case ASSIGNED_TO_CUSTOMER: + case ASSIGNED_TO_CUSTOMER: { CustomerId customerId = JacksonUtil.fromString(edgeNotificationMsg.getBody(), CustomerId.class); - Edge edge = edgeService.findEdgeById(tenantId, edgeId); + Edge edge = edgeCtx.getEdgeService().findEdgeById(tenantId, edgeId); if (customerId != null && (edge == null || customerId.isNullUid())) { return Futures.immediateFuture(null); } @@ -82,7 +80,7 @@ public class EdgeProcessor extends BaseEdgeProcessor { PageLink pageLink = new PageLink(1000); PageData pageData; do { - pageData = userService.findCustomerUsers(tenantId, customerId, pageLink); + pageData = edgeCtx.getUserService().findCustomerUsers(tenantId, customerId, pageLink); if (pageData != null && pageData.getData() != null && !pageData.getData().isEmpty()) { log.trace("[{}][{}][{}] user(s) are going to be added to edge.", tenantId, edge.getId(), pageData.getData().size()); for (User user : pageData.getData()) { @@ -94,15 +92,17 @@ public class EdgeProcessor extends BaseEdgeProcessor { } } while (pageData != null && pageData.hasNext()); return Futures.transform(Futures.allAsList(futures), voids -> null, dbCallbackExecutorService); - case UNASSIGNED_FROM_CUSTOMER: + } + case UNASSIGNED_FROM_CUSTOMER: { CustomerId customerIdToDelete = JacksonUtil.fromString(edgeNotificationMsg.getBody(), CustomerId.class); - edge = edgeService.findEdgeById(tenantId, edgeId); + Edge edge = edgeCtx.getEdgeService().findEdgeById(tenantId, edgeId); if (customerIdToDelete != null && (edge == null || customerIdToDelete.isNullUid())) { return Futures.immediateFuture(null); } return Futures.transformAsync(saveEdgeEvent(edge.getTenantId(), edge.getId(), EdgeEventType.EDGE, EdgeEventActionType.UNASSIGNED_FROM_CUSTOMER, edgeId, null), voids -> saveEdgeEvent(edge.getTenantId(), edge.getId(), EdgeEventType.CUSTOMER, EdgeEventActionType.DELETED, customerIdToDelete, null), dbCallbackExecutorService); + } default: return Futures.immediateFuture(null); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/BaseEntityViewProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/BaseEntityViewProcessor.java index 9c2b9a37bd..757f606f69 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/BaseEntityViewProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/BaseEntityViewProcessor.java @@ -16,18 +16,23 @@ package org.thingsboard.server.service.edge.rpc.processor.entityview; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.util.Pair; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j public abstract class BaseEntityViewProcessor extends BaseEdgeProcessor { + @Autowired + private DataValidator entityViewValidator; + protected Pair saveOrUpdateEntityView(TenantId tenantId, EntityViewId entityViewId, EntityViewUpdateMsg entityViewUpdateMsg) { boolean created = false; boolean entityViewNameUpdated = false; @@ -35,7 +40,7 @@ public abstract class BaseEntityViewProcessor extends BaseEdgeProcessor { if (entityView == null) { throw new RuntimeException("[{" + tenantId + "}] entityViewUpdateMsg {" + entityViewUpdateMsg + "} cannot be converted to entity view"); } - EntityView entityViewById = entityViewService.findEntityViewById(tenantId, entityViewId); + EntityView entityViewById = edgeCtx.getEntityViewService().findEntityViewById(tenantId, entityViewId); if (entityViewById == null) { created = true; entityView.setId(null); @@ -43,7 +48,7 @@ public abstract class BaseEntityViewProcessor extends BaseEdgeProcessor { entityView.setId(entityViewId); } String entityViewName = entityView.getName(); - EntityView entityViewByName = entityViewService.findEntityViewByTenantIdAndName(tenantId, entityViewName); + EntityView entityViewByName = edgeCtx.getEntityViewService().findEntityViewByTenantIdAndName(tenantId, entityViewName); if (entityViewByName != null && !entityViewByName.getId().equals(entityViewId)) { entityViewName = entityViewName + "_" + StringUtils.randomAlphanumeric(15); log.warn("[{}] Entity view with name {} already exists. Renaming entity view name to {}", @@ -57,7 +62,7 @@ public abstract class BaseEntityViewProcessor extends BaseEdgeProcessor { if (created) { entityView.setId(entityViewId); } - entityViewService.saveEntityView(entityView, false); + edgeCtx.getEntityViewService().saveEntityView(entityView, false); return Pair.of(created, entityViewNameUpdated); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java index 4ef37f703b..aa105652d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.edge.rpc.processor.entityview; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.util.Pair; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EdgeUtils; @@ -36,12 +37,16 @@ import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.service.edge.rpc.constructor.entityview.EntityViewMsgConstructor; +import org.thingsboard.server.service.edge.rpc.constructor.entityview.EntityViewMsgConstructorFactory; import java.util.UUID; @Slf4j public abstract class EntityViewEdgeProcessor extends BaseEntityViewProcessor implements EntityViewProcessor { + @Autowired + private EntityViewMsgConstructorFactory entityViewMsgConstructorFactory; + @Override public ListenableFuture processEntityViewMsgFromEdge(TenantId tenantId, Edge edge, EntityViewUpdateMsg entityViewUpdateMsg) { log.trace("[{}] executing processEntityViewMsgFromEdge [{}] from edge [{}]", tenantId, entityViewUpdateMsg, edge.getId()); @@ -55,9 +60,9 @@ public abstract class EntityViewEdgeProcessor extends BaseEntityViewProcessor im saveOrUpdateEntityView(tenantId, entityViewId, entityViewUpdateMsg, edge); return Futures.immediateFuture(null); case ENTITY_DELETED_RPC_MESSAGE: - EntityView entityViewToDelete = entityViewService.findEntityViewById(tenantId, entityViewId); + EntityView entityViewToDelete = edgeCtx.getEntityViewService().findEntityViewById(tenantId, entityViewId); if (entityViewToDelete != null) { - entityViewService.unassignEntityViewFromEdge(tenantId, entityViewId, edge.getId()); + edgeCtx.getEntityViewService().unassignEntityViewFromEdge(tenantId, entityViewId, edge.getId()); } return Futures.immediateFuture(null); case UNRECOGNIZED: @@ -82,7 +87,7 @@ public abstract class EntityViewEdgeProcessor extends BaseEntityViewProcessor im if (created) { createRelationFromEdge(tenantId, edge.getId(), entityViewId); pushEntityViewCreatedEventToRuleEngine(tenantId, edge, entityViewId); - entityViewService.assignEntityViewToEdge(tenantId, entityViewId, edge.getId()); + edgeCtx.getEntityViewService().assignEntityViewToEdge(tenantId, entityViewId, edge.getId()); } Boolean assetNameUpdated = resultPair.getSecond(); if (assetNameUpdated) { @@ -92,7 +97,7 @@ public abstract class EntityViewEdgeProcessor extends BaseEntityViewProcessor im private void pushEntityViewCreatedEventToRuleEngine(TenantId tenantId, Edge edge, EntityViewId entityViewId) { try { - EntityView entityView = entityViewService.findEntityViewById(tenantId, entityViewId); + EntityView entityView = edgeCtx.getEntityViewService().findEntityViewById(tenantId, entityViewId); String entityViewAsString = JacksonUtil.toString(entityView); TbMsgMetaData msgMetaData = getEdgeActionTbMsgMetaData(edge, entityView.getCustomerId()); pushEntityEventToRuleEngine(tenantId, entityViewId, entityView.getCustomerId(), TbMsgType.ENTITY_CREATED, entityViewAsString, msgMetaData); @@ -103,26 +108,28 @@ public abstract class EntityViewEdgeProcessor extends BaseEntityViewProcessor im public DownlinkMsg convertEntityViewEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { EntityViewId entityViewId = new EntityViewId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; var msgConstructor = (EntityViewMsgConstructor) entityViewMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED, ASSIGNED_TO_EDGE, ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER -> { - EntityView entityView = entityViewService.findEntityViewById(edgeEvent.getTenantId(), entityViewId); + EntityView entityView = edgeCtx.getEntityViewService().findEntityViewById(edgeEvent.getTenantId(), entityViewId); if (entityView != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); EntityViewUpdateMsg entityViewUpdateMsg = msgConstructor.constructEntityViewUpdatedMsg(msgType, entityView); - downlinkMsg = DownlinkMsg.newBuilder() + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addEntityViewUpdateMsg(entityViewUpdateMsg) .build(); } } - case DELETED, UNASSIGNED_FROM_EDGE -> downlinkMsg = DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addEntityViewUpdateMsg(msgConstructor.constructEntityViewDeleteMsg(entityViewId)) - .build(); + case DELETED, UNASSIGNED_FROM_EDGE -> { + EntityViewUpdateMsg entityViewUpdateMsg = msgConstructor.constructEntityViewDeleteMsg(entityViewId); + return DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addEntityViewUpdateMsg(entityViewUpdateMsg) + .build(); + } } - return downlinkMsg; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationEdgeProcessor.java index 6af039341f..67aa1dece2 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/notification/NotificationEdgeProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.edge.rpc.processor.notification; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; @@ -25,12 +26,16 @@ import org.thingsboard.server.common.data.id.NotificationTemplateId; import org.thingsboard.server.common.data.notification.rule.NotificationRule; import org.thingsboard.server.common.data.notification.targets.NotificationTarget; import org.thingsboard.server.common.data.notification.template.NotificationTemplate; +import org.thingsboard.server.dao.notification.NotificationRuleService; +import org.thingsboard.server.dao.notification.NotificationTargetService; +import org.thingsboard.server.dao.notification.NotificationTemplateService; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.NotificationRuleUpdateMsg; import org.thingsboard.server.gen.edge.v1.NotificationTargetUpdateMsg; import org.thingsboard.server.gen.edge.v1.NotificationTemplateUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.rpc.constructor.notification.NotificationMsgConstructor; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j @@ -38,6 +43,18 @@ import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @TbCoreComponent public class NotificationEdgeProcessor extends BaseEdgeProcessor { + @Autowired + protected NotificationRuleService notificationRuleService; + + @Autowired + private NotificationTargetService notificationTargetService; + + @Autowired + private NotificationTemplateService notificationTemplateService; + + @Autowired + private NotificationMsgConstructor notificationMsgConstructor; + public DownlinkMsg convertNotificationRuleToDownlink(EdgeEvent edgeEvent) { NotificationRuleId notificationRuleId = new NotificationRuleId(edgeEvent.getEntityId()); DownlinkMsg downlinkMsg = null; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/OAuth2EdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/OAuth2EdgeProcessor.java index b13cf567ef..9f4af7d0dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/OAuth2EdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/oauth2/OAuth2EdgeProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.edge.rpc.processor.oauth2; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.domain.DomainInfo; @@ -23,12 +24,14 @@ import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DomainId; import org.thingsboard.server.common.data.id.OAuth2ClientId; import org.thingsboard.server.common.data.oauth2.OAuth2Client; +import org.thingsboard.server.dao.domain.DomainService; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.OAuth2ClientUpdateMsg; import org.thingsboard.server.gen.edge.v1.OAuth2DomainUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.rpc.constructor.oauth2.OAuth2MsgConstructor; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; import org.thingsboard.server.service.edge.rpc.utils.EdgeVersionUtils; @@ -37,12 +40,17 @@ import org.thingsboard.server.service.edge.rpc.utils.EdgeVersionUtils; @TbCoreComponent public class OAuth2EdgeProcessor extends BaseEdgeProcessor { + @Autowired + private DomainService domainService; + + @Autowired + private OAuth2MsgConstructor oAuth2MsgConstructor; + public DownlinkMsg convertOAuth2DomainEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { if (EdgeVersionUtils.isEdgeVersionOlderThan(edgeVersion, EdgeVersion.V_3_8_0)) { return null; } DomainId domainId = new DomainId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { @@ -54,22 +62,22 @@ public class OAuth2EdgeProcessor extends BaseEdgeProcessor { .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addOAuth2DomainUpdateMsg(oAuth2DomainUpdateMsg); domainInfo.getOauth2ClientInfos().forEach(clientInfo -> { - OAuth2Client oauth2Client = oAuth2ClientService.findOAuth2ClientById(edgeEvent.getTenantId(), clientInfo.getId()); + OAuth2Client oauth2Client = edgeCtx.getOAuth2ClientService().findOAuth2ClientById(edgeEvent.getTenantId(), clientInfo.getId()); OAuth2ClientUpdateMsg oAuth2ClientUpdateMsg = oAuth2MsgConstructor.constructOAuth2ClientUpdateMsg(msgType, oauth2Client); builder.addOAuth2ClientUpdateMsg(oAuth2ClientUpdateMsg); }); - downlinkMsg = builder.build(); + return builder.build(); } } case DELETED -> { OAuth2DomainUpdateMsg oAuth2DomainUpdateMsg = oAuth2MsgConstructor.constructOAuth2DomainDeleteMsg(domainId); - downlinkMsg = DownlinkMsg.newBuilder() + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addOAuth2DomainUpdateMsg(oAuth2DomainUpdateMsg) .build(); } } - return downlinkMsg; + return null; } public DownlinkMsg convertOAuth2ClientEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { @@ -77,19 +85,18 @@ public class OAuth2EdgeProcessor extends BaseEdgeProcessor { return null; } OAuth2ClientId oAuth2ClientId = new OAuth2ClientId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { - boolean isPropagateToEdge = oAuth2ClientService.isPropagateOAuth2ClientToEdge(edgeEvent.getTenantId(), oAuth2ClientId); + boolean isPropagateToEdge = edgeCtx.getOAuth2ClientService().isPropagateOAuth2ClientToEdge(edgeEvent.getTenantId(), oAuth2ClientId); if (!isPropagateToEdge) { return null; } - OAuth2Client oAuth2Client = oAuth2ClientService.findOAuth2ClientById(edgeEvent.getTenantId(), oAuth2ClientId); + OAuth2Client oAuth2Client = edgeCtx.getOAuth2ClientService().findOAuth2ClientById(edgeEvent.getTenantId(), oAuth2ClientId); if (oAuth2Client != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); OAuth2ClientUpdateMsg oAuth2ClientUpdateMsg = oAuth2MsgConstructor.constructOAuth2ClientUpdateMsg(msgType, oAuth2Client); - downlinkMsg = DownlinkMsg.newBuilder() + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addOAuth2ClientUpdateMsg(oAuth2ClientUpdateMsg) .build(); @@ -97,13 +104,13 @@ public class OAuth2EdgeProcessor extends BaseEdgeProcessor { } case DELETED -> { OAuth2ClientUpdateMsg oAuth2ClientDeleteMsg = oAuth2MsgConstructor.constructOAuth2ClientDeleteMsg(oAuth2ClientId); - downlinkMsg = DownlinkMsg.newBuilder() + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addOAuth2ClientUpdateMsg(oAuth2ClientDeleteMsg) .build(); } } - return downlinkMsg; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/ota/OtaPackageEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/ota/OtaPackageEdgeProcessor.java index 34a7724fee..fc803caf4d 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/ota/OtaPackageEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/ota/OtaPackageEdgeProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.edge.rpc.processor.ota; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.OtaPackage; @@ -27,6 +28,7 @@ import org.thingsboard.server.gen.edge.v1.OtaPackageUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.rpc.constructor.ota.OtaPackageMsgConstructor; +import org.thingsboard.server.service.edge.rpc.constructor.ota.OtaPackageMsgConstructorFactory; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Component @@ -34,32 +36,33 @@ import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @TbCoreComponent public class OtaPackageEdgeProcessor extends BaseEdgeProcessor { + @Autowired + private OtaPackageMsgConstructorFactory otaPackageMsgConstructorFactory; + public DownlinkMsg convertOtaPackageEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { OtaPackageId otaPackageId = new OtaPackageId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; + var msgConstructor = (OtaPackageMsgConstructor) otaPackageMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { - OtaPackage otaPackage = otaPackageService.findOtaPackageById(edgeEvent.getTenantId(), otaPackageId); + OtaPackage otaPackage = edgeCtx.getOtaPackageService().findOtaPackageById(edgeEvent.getTenantId(), otaPackageId); if (otaPackage != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - OtaPackageUpdateMsg otaPackageUpdateMsg = ((OtaPackageMsgConstructor) - otaPackageMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructOtaPackageUpdatedMsg(msgType, otaPackage); - downlinkMsg = DownlinkMsg.newBuilder() + OtaPackageUpdateMsg otaPackageUpdateMsg = msgConstructor.constructOtaPackageUpdatedMsg(msgType, otaPackage); + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addOtaPackageUpdateMsg(otaPackageUpdateMsg) .build(); } } case DELETED -> { - OtaPackageUpdateMsg otaPackageUpdateMsg = ((OtaPackageMsgConstructor) - otaPackageMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructOtaPackageDeleteMsg(otaPackageId); - downlinkMsg = DownlinkMsg.newBuilder() + OtaPackageUpdateMsg otaPackageUpdateMsg = msgConstructor.constructOtaPackageDeleteMsg(otaPackageId); + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addOtaPackageUpdateMsg(otaPackageUpdateMsg) .build(); } } - return downlinkMsg; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/queue/QueueEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/queue/QueueEdgeProcessor.java index bfa81361fa..76f068ac7a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/queue/QueueEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/queue/QueueEdgeProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.edge.rpc.processor.queue; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; @@ -27,6 +28,7 @@ import org.thingsboard.server.gen.edge.v1.QueueUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.rpc.constructor.queue.QueueMsgConstructor; +import org.thingsboard.server.service.edge.rpc.constructor.queue.QueueMsgConstructorFactory; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j @@ -34,32 +36,33 @@ import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @TbCoreComponent public class QueueEdgeProcessor extends BaseEdgeProcessor { + @Autowired + private QueueMsgConstructorFactory queueMsgConstructorFactory; + public DownlinkMsg convertQueueEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { QueueId queueId = new QueueId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; + var msgConstructor = (QueueMsgConstructor) queueMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { - Queue queue = queueService.findQueueById(edgeEvent.getTenantId(), queueId); + Queue queue = edgeCtx.getQueueService().findQueueById(edgeEvent.getTenantId(), queueId); if (queue != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - QueueUpdateMsg queueUpdateMsg = ((QueueMsgConstructor) - queueMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructQueueUpdatedMsg(msgType, queue); - downlinkMsg = DownlinkMsg.newBuilder() + QueueUpdateMsg queueUpdateMsg = msgConstructor.constructQueueUpdatedMsg(msgType, queue); + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addQueueUpdateMsg(queueUpdateMsg) .build(); } } case DELETED -> { - QueueUpdateMsg queueDeleteMsg = ((QueueMsgConstructor) - queueMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructQueueDeleteMsg(queueId); - downlinkMsg = DownlinkMsg.newBuilder() + QueueUpdateMsg queueDeleteMsg = msgConstructor.constructQueueDeleteMsg(queueId); + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addQueueUpdateMsg(queueDeleteMsg) .build(); } } - return downlinkMsg; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/BaseRelationProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/BaseRelationProcessor.java index a832f0b1e1..9ba863f62a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/BaseRelationProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/BaseRelationProcessor.java @@ -38,13 +38,13 @@ public abstract class BaseRelationProcessor extends BaseEdgeProcessor { case ENTITY_UPDATED_RPC_MESSAGE: if (isEntityExists(tenantId, entityRelation.getTo()) && isEntityExists(tenantId, entityRelation.getFrom())) { - relationService.saveRelation(tenantId, entityRelation); + edgeCtx.getRelationService().saveRelation(tenantId, entityRelation); } else { log.warn("[{}] Skipping relating update msg because from/to entity doesn't exists on edge, {}", tenantId, relationUpdateMsg); } break; case ENTITY_DELETED_RPC_MESSAGE: - relationService.deleteRelation(tenantId, entityRelation); + edgeCtx.getRelationService().deleteRelation(tenantId, entityRelation); break; case UNRECOGNIZED: default: diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessor.java index 780550ed95..87b9797620 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessor.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.edge.rpc.processor.relation; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.EntityType; @@ -34,6 +35,7 @@ import org.thingsboard.server.gen.edge.v1.RelationUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.edge.rpc.constructor.relation.RelationMsgConstructor; +import org.thingsboard.server.service.edge.rpc.constructor.relation.RelationMsgConstructorFactory; import java.util.ArrayList; import java.util.HashSet; @@ -43,6 +45,9 @@ import java.util.Set; @Slf4j public abstract class RelationEdgeProcessor extends BaseRelationProcessor implements RelationProcessor { + @Autowired + private RelationMsgConstructorFactory relationMsgConstructorFactory; + @Override public ListenableFuture processRelationMsgFromEdge(TenantId tenantId, Edge edge, RelationUpdateMsg relationUpdateMsg) { log.trace("[{}] executing processRelationMsgFromEdge [{}] from edge [{}]", tenantId, relationUpdateMsg, edge.getId()); @@ -74,8 +79,8 @@ public abstract class RelationEdgeProcessor extends BaseRelationProcessor implem EdgeId originatorEdgeId = safeGetEdgeId(edgeNotificationMsg.getOriginatorEdgeIdMSB(), edgeNotificationMsg.getOriginatorEdgeIdLSB()); Set uniqueEdgeIds = new HashSet<>(); - uniqueEdgeIds.addAll(edgeService.findAllRelatedEdgeIds(tenantId, relation.getTo())); - uniqueEdgeIds.addAll(edgeService.findAllRelatedEdgeIds(tenantId, relation.getFrom())); + uniqueEdgeIds.addAll(edgeCtx.getEdgeService().findAllRelatedEdgeIds(tenantId, relation.getTo())); + uniqueEdgeIds.addAll(edgeCtx.getEdgeService().findAllRelatedEdgeIds(tenantId, relation.getFrom())); uniqueEdgeIds.remove(originatorEdgeId); if (uniqueEdgeIds.isEmpty()) { return Futures.immediateFuture(null); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/BaseResourceProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/BaseResourceProcessor.java index 7384130baf..ac7bd6768c 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/BaseResourceProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/BaseResourceProcessor.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.edge.rpc.processor.resource; import com.datastax.oss.driver.api.core.uuid.Uuids; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TbResource; @@ -24,12 +25,16 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.id.TbResourceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j public abstract class BaseResourceProcessor extends BaseEdgeProcessor { + @Autowired + private DataValidator resourceValidator; + protected boolean saveOrUpdateTbResource(TenantId tenantId, TbResourceId tbResourceId, ResourceUpdateMsg resourceUpdateMsg) { boolean resourceKeyUpdated = false; try { @@ -38,7 +43,7 @@ public abstract class BaseResourceProcessor extends BaseEdgeProcessor { throw new RuntimeException("[{" + tenantId + "}] resourceUpdateMsg {" + resourceUpdateMsg + " } cannot be converted to resource"); } boolean created = false; - TbResource resourceById = resourceService.findResourceById(tenantId, tbResourceId); + TbResource resourceById = edgeCtx.getResourceService().findResourceById(tenantId, tbResourceId); if (resourceById == null) { resource.setCreatedTime(Uuids.unixTimestamp(tbResourceId.getId())); created = true; @@ -49,7 +54,7 @@ public abstract class BaseResourceProcessor extends BaseEdgeProcessor { String resourceKey = resource.getResourceKey(); ResourceType resourceType = resource.getResourceType(); PageDataIterable resourcesIterable = new PageDataIterable<>( - link -> resourceService.findTenantResourcesByResourceTypeAndPageLink(tenantId, resourceType, link), 1024); + link -> edgeCtx.getResourceService().findTenantResourcesByResourceTypeAndPageLink(tenantId, resourceType, link), 1024); for (TbResource tbResource : resourcesIterable) { if (tbResource.getResourceKey().equals(resourceKey) && !tbResourceId.equals(tbResource.getId())) { resourceKey = StringUtils.randomAlphabetic(15) + "_" + resourceKey; @@ -63,7 +68,7 @@ public abstract class BaseResourceProcessor extends BaseEdgeProcessor { if (created) { resource.setId(tbResourceId); } - resourceService.saveResource(resource, false); + edgeCtx.getResourceService().saveResource(resource, false); } catch (Exception e) { log.error("[{}] Failed to process resource update msg [{}]", tenantId, resourceUpdateMsg, e); throw e; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java index 0a83e439b5..10c58e89d0 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.edge.rpc.processor.resource; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.edge.Edge; @@ -32,12 +33,16 @@ import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.service.edge.rpc.constructor.resource.ResourceMsgConstructor; +import org.thingsboard.server.service.edge.rpc.constructor.resource.ResourceMsgConstructorFactory; import java.util.UUID; @Slf4j public abstract class ResourceEdgeProcessor extends BaseResourceProcessor implements ResourceProcessor { + @Autowired + private ResourceMsgConstructorFactory resourceMsgConstructorFactory; + @Override public ListenableFuture processResourceMsgFromEdge(TenantId tenantId, Edge edge, ResourceUpdateMsg resourceUpdateMsg) { TbResourceId tbResourceId = new TbResourceId(new UUID(resourceUpdateMsg.getIdMSB(), resourceUpdateMsg.getIdLSB())); @@ -72,30 +77,28 @@ public abstract class ResourceEdgeProcessor extends BaseResourceProcessor implem @Override public DownlinkMsg convertResourceEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { TbResourceId tbResourceId = new TbResourceId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; + var msgConstructor = (ResourceMsgConstructor) resourceMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { - TbResource tbResource = resourceService.findResourceById(edgeEvent.getTenantId(), tbResourceId); + TbResource tbResource = edgeCtx.getResourceService().findResourceById(edgeEvent.getTenantId(), tbResourceId); if (tbResource != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - ResourceUpdateMsg resourceUpdateMsg = ((ResourceMsgConstructor) - resourceMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructResourceUpdatedMsg(msgType, tbResource); - downlinkMsg = resourceUpdateMsg != null ? DownlinkMsg.newBuilder() + ResourceUpdateMsg resourceUpdateMsg = msgConstructor.constructResourceUpdatedMsg(msgType, tbResource); + return resourceUpdateMsg != null ? DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addResourceUpdateMsg(resourceUpdateMsg) .build() : null; } } case DELETED -> { - ResourceUpdateMsg resourceUpdateMsg = ((ResourceMsgConstructor) - resourceMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructResourceDeleteMsg(tbResourceId); - downlinkMsg = DownlinkMsg.newBuilder() + ResourceUpdateMsg resourceUpdateMsg = msgConstructor.constructResourceDeleteMsg(tbResourceId); + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addResourceUpdateMsg(resourceUpdateMsg) .build(); } } - return downlinkMsg; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java index 310d8d9b9b..6df069e400 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.edge.rpc.processor.rule; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.Edge; @@ -30,6 +31,7 @@ import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.rpc.constructor.rule.RuleChainMsgConstructor; +import org.thingsboard.server.service.edge.rpc.constructor.rule.RuleChainMsgConstructorFactory; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; import static org.thingsboard.server.dao.edge.EdgeServiceImpl.EDGE_IS_ROOT_BODY_KEY; @@ -39,13 +41,16 @@ import static org.thingsboard.server.dao.edge.EdgeServiceImpl.EDGE_IS_ROOT_BODY_ @TbCoreComponent public class RuleChainEdgeProcessor extends BaseEdgeProcessor { + @Autowired + private RuleChainMsgConstructorFactory ruleChainMsgConstructorFactory; + public DownlinkMsg convertRuleChainEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { RuleChainId ruleChainId = new RuleChainId(edgeEvent.getEntityId()); DownlinkMsg downlinkMsg = null; var msgConstructor = (RuleChainMsgConstructor) ruleChainMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED, ASSIGNED_TO_EDGE -> { - RuleChain ruleChain = ruleChainService.findRuleChainById(edgeEvent.getTenantId(), ruleChainId); + RuleChain ruleChain = edgeCtx.getRuleChainService().findRuleChainById(edgeEvent.getTenantId(), ruleChainId); if (ruleChain != null) { boolean isRoot = false; if (edgeEvent.getBody() != null && edgeEvent.getBody().get(EDGE_IS_ROOT_BODY_KEY) != null) { @@ -55,7 +60,7 @@ public class RuleChainEdgeProcessor extends BaseEdgeProcessor { } } if (!isRoot) { - Edge edge = edgeService.findEdgeById(edgeEvent.getTenantId(), edgeEvent.getEdgeId()); + Edge edge = edgeCtx.getEdgeService().findEdgeById(edgeEvent.getTenantId(), edgeEvent.getEdgeId()); isRoot = edge.getRootRuleChainId().equals(ruleChainId); } UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); @@ -65,9 +70,8 @@ public class RuleChainEdgeProcessor extends BaseEdgeProcessor { .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addRuleChainUpdateMsg(ruleChainUpdateMsg); - RuleChainMetaData ruleChainMetaData = ruleChainService.loadRuleChainMetaData(edgeEvent.getTenantId(), ruleChainId); - RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = ((RuleChainMsgConstructor) - ruleChainMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)) + RuleChainMetaData ruleChainMetaData = edgeCtx.getRuleChainService().loadRuleChainMetaData(edgeEvent.getTenantId(), ruleChainId); + RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = msgConstructor .constructRuleChainMetadataUpdatedMsg(edgeEvent.getTenantId(), msgType, ruleChainMetaData, edgeVersion); if (ruleChainMetadataUpdateMsg != null) { builder.addRuleChainMetadataUpdateMsg(ruleChainMetadataUpdateMsg); @@ -85,22 +89,21 @@ public class RuleChainEdgeProcessor extends BaseEdgeProcessor { public DownlinkMsg convertRuleChainMetadataEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { RuleChainId ruleChainId = new RuleChainId(edgeEvent.getEntityId()); - RuleChain ruleChain = ruleChainService.findRuleChainById(edgeEvent.getTenantId(), ruleChainId); - DownlinkMsg downlinkMsg = null; + RuleChain ruleChain = edgeCtx.getRuleChainService().findRuleChainById(edgeEvent.getTenantId(), ruleChainId); var msgConstructor = (RuleChainMsgConstructor) ruleChainMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); if (ruleChain != null) { - RuleChainMetaData ruleChainMetaData = ruleChainService.loadRuleChainMetaData(edgeEvent.getTenantId(), ruleChainId); + RuleChainMetaData ruleChainMetaData = edgeCtx.getRuleChainService().loadRuleChainMetaData(edgeEvent.getTenantId(), ruleChainId); UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = msgConstructor .constructRuleChainMetadataUpdatedMsg(edgeEvent.getTenantId(), msgType, ruleChainMetaData, edgeVersion); if (ruleChainMetadataUpdateMsg != null) { - downlinkMsg = DownlinkMsg.newBuilder() + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addRuleChainMetadataUpdateMsg(ruleChainMetadataUpdateMsg) .build(); } } - return downlinkMsg; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/settings/AdminSettingsEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/settings/AdminSettingsEdgeProcessor.java index d9b0a85056..3c0d4fb99c 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/settings/AdminSettingsEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/settings/AdminSettingsEdgeProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.edge.rpc.processor.settings; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AdminSettings; @@ -26,6 +27,7 @@ import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.rpc.constructor.settings.AdminSettingsMsgConstructor; +import org.thingsboard.server.service.edge.rpc.constructor.settings.AdminSettingsMsgConstructorFactory; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j @@ -33,6 +35,9 @@ import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @TbCoreComponent public class AdminSettingsEdgeProcessor extends BaseEdgeProcessor { + @Autowired + private AdminSettingsMsgConstructorFactory adminSettingsMsgConstructorFactory; + public DownlinkMsg convertAdminSettingsEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { AdminSettings adminSettings = JacksonUtil.convertValue(edgeEvent.getBody(), AdminSettings.class); if (adminSettings == null) { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java index 2656ef622f..d94d4d8939 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java @@ -28,6 +28,8 @@ import jakarta.annotation.PostConstruct; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.common.data.AttributeScope; @@ -66,7 +68,13 @@ import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.service.edge.rpc.constructor.telemetry.EntityDataMsgConstructor; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.util.ArrayList; import java.util.List; @@ -75,6 +83,25 @@ import java.util.UUID; @Slf4j public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { + @Autowired + private EntityDataMsgConstructor entityDataMsgConstructor; + + @Autowired + private PartitionService partitionService; + + @Autowired + private TelemetrySubscriptionService tsSubService; + + @Autowired + private TbDeviceProfileCache deviceProfileCache; + + @Autowired + private TbAssetProfileCache assetProfileCache; + + @Lazy + @Autowired + private TbQueueProducerProvider producerProvider; + private final Gson gson = new Gson(); private TbQueueProducer> tbCoreMsgProducer; @@ -139,7 +166,7 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { CustomerId customerId = null; switch (entityId.getEntityType()) { case DEVICE -> { - Device device = deviceService.findDeviceById(tenantId, new DeviceId(entityId.getId())); + Device device = edgeCtx.getDeviceService().findDeviceById(tenantId, new DeviceId(entityId.getId())); if (device != null) { customerId = device.getCustomerId(); metaData.putValue("deviceName", device.getName()); @@ -147,7 +174,7 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { } } case ASSET -> { - Asset asset = assetService.findAssetById(tenantId, new AssetId(entityId.getId())); + Asset asset = edgeCtx.getAssetService().findAssetById(tenantId, new AssetId(entityId.getId())); if (asset != null) { customerId = asset.getCustomerId(); metaData.putValue("assetName", asset.getName()); @@ -155,7 +182,7 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { } } case ENTITY_VIEW -> { - EntityView entityView = entityViewService.findEntityViewById(tenantId, new EntityViewId(entityId.getId())); + EntityView entityView = edgeCtx.getEntityViewService().findEntityViewById(tenantId, new EntityViewId(entityId.getId())); if (entityView != null) { customerId = entityView.getCustomerId(); metaData.putValue("entityViewName", entityView.getName()); @@ -163,7 +190,7 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { } } case EDGE -> { - Edge edge = edgeService.findEdgeById(tenantId, new EdgeId(entityId.getId())); + Edge edge = edgeCtx.getEdgeService().findEdgeById(tenantId, new EdgeId(entityId.getId())); if (edge != null) { customerId = edge.getCustomerId(); metaData.putValue("edgeName", edge.getName()); @@ -182,7 +209,7 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { metaData.putValue("ts", tsKv.getTs() + ""); var defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId); TbMsg tbMsg = TbMsg.newMsg(defaultQueueAndRuleChain.getKey(), TbMsgType.POST_TELEMETRY_REQUEST, entityId, customerId, metaData, gson.toJson(json), defaultQueueAndRuleChain.getValue(), null); - tbClusterService.pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() { + edgeCtx.getClusterService().pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { futureToSet.set(null); @@ -226,7 +253,7 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { JsonObject json = JsonUtils.getJsonObject(msg.getKvList()); var defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId); TbMsg tbMsg = TbMsg.newMsg(defaultQueueAndRuleChain.getKey(), TbMsgType.POST_ATTRIBUTES_REQUEST, entityId, customerId, metaData, gson.toJson(json), defaultQueueAndRuleChain.getValue(), null); - tbClusterService.pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() { + edgeCtx.getClusterService().pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { futureToSet.set(null); @@ -256,7 +283,7 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { var defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId); TbMsg tbMsg = TbMsg.newMsg(defaultQueueAndRuleChain.getKey(), TbMsgType.ATTRIBUTES_UPDATED, entityId, customerId, metaData, gson.toJson(json), defaultQueueAndRuleChain.getValue(), null); - tbClusterService.pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() { + edgeCtx.getClusterService().pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { futureToSet.set(null); @@ -284,11 +311,11 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { String scope = attributeDeleteMsg.getScope(); List attributeKeys = attributeDeleteMsg.getAttributeNamesList(); - ListenableFuture> removeAllFuture = attributesService.removeAll(tenantId, entityId, AttributeScope.valueOf(scope), attributeKeys); + ListenableFuture> removeAllFuture = edgeCtx.getAttributesService().removeAll(tenantId, entityId, AttributeScope.valueOf(scope), attributeKeys); return Futures.transformAsync(removeAllFuture, removeAttributes -> { if (EntityType.DEVICE.name().equals(entityType)) { SettableFuture futureToSet = SettableFuture.create(); - tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onDelete( + edgeCtx.getClusterService().pushMsgToCore(DeviceAttributesEventNotificationMsg.onDelete( tenantId, (DeviceId) entityId, scope, attributeKeys), new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantEdgeProcessor.java index 515dccb987..86bddd8ea2 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantEdgeProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.edge.rpc.processor.tenant; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.Tenant; @@ -30,6 +31,7 @@ import org.thingsboard.server.gen.edge.v1.TenantUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.rpc.constructor.tenant.TenantMsgConstructor; +import org.thingsboard.server.service.edge.rpc.constructor.tenant.TenantMsgConstructorFactory; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j @@ -37,28 +39,27 @@ import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @TbCoreComponent public class TenantEdgeProcessor extends BaseEdgeProcessor { + @Autowired + private TenantMsgConstructorFactory tenantMsgConstructorFactory; + public DownlinkMsg convertTenantEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { TenantId tenantId = new TenantId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; + var msgConstructor = ((TenantMsgConstructor) tenantMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)); if (EdgeEventActionType.UPDATED.equals(edgeEvent.getAction())) { - Tenant tenant = tenantService.findTenantById(tenantId); + Tenant tenant = edgeCtx.getTenantService().findTenantById(tenantId); if (tenant != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - TenantUpdateMsg tenantUpdateMsg = ((TenantMsgConstructor) - tenantMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)) - .constructTenantUpdateMsg(msgType, tenant); - TenantProfile tenantProfile = tenantProfileService.findTenantProfileById(tenantId, tenant.getTenantProfileId()); - TenantProfileUpdateMsg tenantProfileUpdateMsg = ((TenantMsgConstructor) - tenantMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)) - .constructTenantProfileUpdateMsg(msgType, tenantProfile, edgeVersion); - downlinkMsg = DownlinkMsg.newBuilder() + TenantUpdateMsg tenantUpdateMsg = msgConstructor.constructTenantUpdateMsg(msgType, tenant); + TenantProfile tenantProfile = edgeCtx.getTenantProfileService().findTenantProfileById(tenantId, tenant.getTenantProfileId()); + TenantProfileUpdateMsg tenantProfileUpdateMsg = msgConstructor.constructTenantProfileUpdateMsg(msgType, tenantProfile, edgeVersion); + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addTenantUpdateMsg(tenantUpdateMsg) .addTenantProfileUpdateMsg(tenantProfileUpdateMsg) .build(); } } - return downlinkMsg; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantProfileEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantProfileEdgeProcessor.java index 6c3bc1733f..07ba6039af 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantProfileEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/tenant/TenantProfileEdgeProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.edge.rpc.processor.tenant; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.TenantProfile; @@ -28,6 +29,7 @@ import org.thingsboard.server.gen.edge.v1.TenantProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.rpc.constructor.tenant.TenantMsgConstructor; +import org.thingsboard.server.service.edge.rpc.constructor.tenant.TenantMsgConstructorFactory; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j @@ -35,23 +37,24 @@ import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @TbCoreComponent public class TenantProfileEdgeProcessor extends BaseEdgeProcessor { + @Autowired + private TenantMsgConstructorFactory tenantMsgConstructorFactory; + public DownlinkMsg convertTenantProfileEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { TenantProfileId tenantProfileId = new TenantProfileId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; + var msgConstructor = ((TenantMsgConstructor) tenantMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)); if (EdgeEventActionType.UPDATED.equals(edgeEvent.getAction())) { - TenantProfile tenantProfile = tenantProfileService.findTenantProfileById(edgeEvent.getTenantId(), tenantProfileId); + TenantProfile tenantProfile = edgeCtx.getTenantProfileService().findTenantProfileById(edgeEvent.getTenantId(), tenantProfileId); if (tenantProfile != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - TenantProfileUpdateMsg tenantProfileUpdateMsg = ((TenantMsgConstructor) - tenantMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)) - .constructTenantProfileUpdateMsg(msgType, tenantProfile, edgeVersion); - downlinkMsg = DownlinkMsg.newBuilder() + TenantProfileUpdateMsg tenantProfileUpdateMsg = msgConstructor.constructTenantProfileUpdateMsg(msgType, tenantProfile, edgeVersion); + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addTenantProfileUpdateMsg(tenantProfileUpdateMsg) .build(); } } - return downlinkMsg; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java index cd072b4a1e..319812a33b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.edge.rpc.processor.user; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.User; @@ -28,6 +29,7 @@ import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.edge.v1.UserCredentialsUpdateMsg; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.rpc.constructor.user.UserMsgConstructor; +import org.thingsboard.server.service.edge.rpc.constructor.user.UserMsgConstructorFactory; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j @@ -35,43 +37,45 @@ import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @TbCoreComponent public class UserEdgeProcessor extends BaseEdgeProcessor { + @Autowired + private UserMsgConstructorFactory userMsgConstructorFactory; + public DownlinkMsg convertUserEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { UserId userId = new UserId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; + var msgConstructor = (UserMsgConstructor) userMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { - User user = userService.findUserById(edgeEvent.getTenantId(), userId); + User user = edgeCtx.getUserService().findUserById(edgeEvent.getTenantId(), userId); if (user != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); DownlinkMsg.Builder builder = DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addUserUpdateMsg(((UserMsgConstructor) userMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructUserUpdatedMsg(msgType, user)); - UserCredentials userCredentialsByUserId = userService.findUserCredentialsByUserId(edgeEvent.getTenantId(), userId); + .addUserUpdateMsg(msgConstructor.constructUserUpdatedMsg(msgType, user)); + UserCredentials userCredentialsByUserId = edgeCtx.getUserService().findUserCredentialsByUserId(edgeEvent.getTenantId(), userId); if (userCredentialsByUserId != null && userCredentialsByUserId.isEnabled()) { - UserCredentialsUpdateMsg userCredentialsUpdateMsg = - ((UserMsgConstructor) userMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructUserCredentialsUpdatedMsg(userCredentialsByUserId); - builder.addUserCredentialsUpdateMsg(userCredentialsUpdateMsg); + builder.addUserCredentialsUpdateMsg(msgConstructor.constructUserCredentialsUpdatedMsg(userCredentialsByUserId)); } - downlinkMsg = builder.build(); + return builder.build(); } } - case DELETED -> downlinkMsg = DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addUserUpdateMsg(((UserMsgConstructor) userMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructUserDeleteMsg(userId)) - .build(); + case DELETED -> { + return DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addUserUpdateMsg(msgConstructor.constructUserDeleteMsg(userId)) + .build(); + } case CREDENTIALS_UPDATED -> { - UserCredentials userCredentialsByUserId = userService.findUserCredentialsByUserId(edgeEvent.getTenantId(), userId); + UserCredentials userCredentialsByUserId = edgeCtx.getUserService().findUserCredentialsByUserId(edgeEvent.getTenantId(), userId); if (userCredentialsByUserId != null && userCredentialsByUserId.isEnabled()) { - UserCredentialsUpdateMsg userCredentialsUpdateMsg = - ((UserMsgConstructor) userMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructUserCredentialsUpdatedMsg(userCredentialsByUserId); - downlinkMsg = DownlinkMsg.newBuilder() + UserCredentialsUpdateMsg userCredentialsUpdateMsg = msgConstructor.constructUserCredentialsUpdatedMsg(userCredentialsByUserId); + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addUserCredentialsUpdateMsg(userCredentialsUpdateMsg) .build(); } } } - return downlinkMsg; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetBundleEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetBundleEdgeProcessor.java index 662b3677cc..87eece2841 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetBundleEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetBundleEdgeProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.edge.rpc.processor.widget; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; @@ -27,6 +28,7 @@ import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.edge.v1.WidgetsBundleUpdateMsg; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.rpc.constructor.widget.WidgetMsgConstructor; +import org.thingsboard.server.service.edge.rpc.constructor.widget.WidgetMsgConstructorFactory; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; import java.util.List; @@ -36,33 +38,34 @@ import java.util.List; @TbCoreComponent public class WidgetBundleEdgeProcessor extends BaseEdgeProcessor { + @Autowired + private WidgetMsgConstructorFactory widgetMsgConstructorFactory; + public DownlinkMsg convertWidgetsBundleEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { WidgetsBundleId widgetsBundleId = new WidgetsBundleId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; + var msgConstructor = ((WidgetMsgConstructor) widgetMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)); switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { - WidgetsBundle widgetsBundle = widgetsBundleService.findWidgetsBundleById(edgeEvent.getTenantId(), widgetsBundleId); + WidgetsBundle widgetsBundle = edgeCtx.getWidgetsBundleService().findWidgetsBundleById(edgeEvent.getTenantId(), widgetsBundleId); if (widgetsBundle != null) { - List widgets = widgetTypeService.findWidgetFqnsByWidgetsBundleId(edgeEvent.getTenantId(), widgetsBundleId); + List widgets = edgeCtx.getWidgetTypeService().findWidgetFqnsByWidgetsBundleId(edgeEvent.getTenantId(), widgetsBundleId); UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - WidgetsBundleUpdateMsg widgetsBundleUpdateMsg = - ((WidgetMsgConstructor) widgetMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructWidgetsBundleUpdateMsg(msgType, widgetsBundle, widgets); - downlinkMsg = DownlinkMsg.newBuilder() + WidgetsBundleUpdateMsg widgetsBundleUpdateMsg = msgConstructor.constructWidgetsBundleUpdateMsg(msgType, widgetsBundle, widgets); + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addWidgetsBundleUpdateMsg(widgetsBundleUpdateMsg) .build(); } } case DELETED -> { - WidgetsBundleUpdateMsg widgetsBundleUpdateMsg = - ((WidgetMsgConstructor) widgetMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructWidgetsBundleDeleteMsg(widgetsBundleId); - downlinkMsg = DownlinkMsg.newBuilder() + WidgetsBundleUpdateMsg widgetsBundleUpdateMsg = msgConstructor.constructWidgetsBundleDeleteMsg(widgetsBundleId); + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addWidgetsBundleUpdateMsg(widgetsBundleUpdateMsg) .build(); } } - return downlinkMsg; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetTypeEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetTypeEdgeProcessor.java index c1891b8fdb..10d2ce4e82 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetTypeEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/widget/WidgetTypeEdgeProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.edge.rpc.processor.widget; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; @@ -27,6 +28,7 @@ import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.edge.v1.WidgetTypeUpdateMsg; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.rpc.constructor.widget.WidgetMsgConstructor; +import org.thingsboard.server.service.edge.rpc.constructor.widget.WidgetMsgConstructorFactory; import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @Slf4j @@ -34,32 +36,33 @@ import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; @TbCoreComponent public class WidgetTypeEdgeProcessor extends BaseEdgeProcessor { + @Autowired + private WidgetMsgConstructorFactory widgetMsgConstructorFactory; + public DownlinkMsg convertWidgetTypeEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { WidgetTypeId widgetTypeId = new WidgetTypeId(edgeEvent.getEntityId()); - DownlinkMsg downlinkMsg = null; + var msgConstructor = (WidgetMsgConstructor) widgetMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion); switch (edgeEvent.getAction()) { case ADDED, UPDATED -> { - WidgetTypeDetails widgetTypeDetails = widgetTypeService.findWidgetTypeDetailsById(edgeEvent.getTenantId(), widgetTypeId); + WidgetTypeDetails widgetTypeDetails = edgeCtx.getWidgetTypeService().findWidgetTypeDetailsById(edgeEvent.getTenantId(), widgetTypeId); if (widgetTypeDetails != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); - WidgetTypeUpdateMsg widgetTypeUpdateMsg = - ((WidgetMsgConstructor) widgetMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructWidgetTypeUpdateMsg(msgType, widgetTypeDetails, edgeVersion); - downlinkMsg = DownlinkMsg.newBuilder() + WidgetTypeUpdateMsg widgetTypeUpdateMsg = msgConstructor.constructWidgetTypeUpdateMsg(msgType, widgetTypeDetails, edgeVersion); + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addWidgetTypeUpdateMsg(widgetTypeUpdateMsg) .build(); } } case DELETED -> { - WidgetTypeUpdateMsg widgetTypeUpdateMsg = - ((WidgetMsgConstructor) widgetMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructWidgetTypeDeleteMsg(widgetTypeId); - downlinkMsg = DownlinkMsg.newBuilder() + WidgetTypeUpdateMsg widgetTypeUpdateMsg = msgConstructor.constructWidgetTypeDeleteMsg(widgetTypeId); + return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addWidgetTypeUpdateMsg(widgetTypeUpdateMsg) .build(); } } - return downlinkMsg; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 2a64dd3388..8e4af4fa28 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -236,7 +236,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, EdgeQueueConfig>builder() .queueKey(new QueueKey(ServiceType.TB_CORE).withQueueName(DataConstants.EDGE_QUEUE_NAME)) - .config(EdgeQueueConfig.of(consumerPerPartition, (int) pollInterval)) + .config(EdgeQueueConfig.of(consumerPerPartition, pollInterval)) .msgPackProcessor(this::processMsgs) .consumerCreator((config, partitionId) -> queueFactory.createEdgeMsgConsumer()) .consumerExecutor(consumersExecutor) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index e9c8668958..9a7994e92a 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1439,7 +1439,7 @@ swagger: # Queue configuration parameters queue: - type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) + type: "${TB_QUEUE_TYPE:kafka}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). in_memory: stats: diff --git a/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java index 5484c29305..46a5d81d7f 100644 --- a/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java @@ -125,7 +125,7 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { protected EdgeImitator edgeImitator; protected Edge edge; - private Random random = new Random(); + private final Random random = new Random(); @Autowired protected EdgeEventService edgeEventService; diff --git a/application/src/test/resources/logback-test.xml b/application/src/test/resources/logback-test.xml index f43144a24e..13c93da411 100644 --- a/application/src/test/resources/logback-test.xml +++ b/application/src/test/resources/logback-test.xml @@ -19,7 +19,7 @@ - + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java index aef3751f0c..c56e4c98d7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java @@ -141,5 +141,6 @@ public class DataConstants { public static final String QUEUE_NAME = "queueName"; public static final String EDGE_QUEUE_NAME = "Edge"; + public static final String EDGE_EVENT_QUEUE_NAME = "EdgeEvent"; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEvent.java index f8e59b308b..9c1449b749 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEvent.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.EdgeEventId; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; import java.util.UUID; @Data @@ -31,6 +32,7 @@ import java.util.UUID; @ToString(callSuper = true) public class EdgeEvent extends BaseData { + @Serial private static final long serialVersionUID = 5548866356798094088L; private long seqId; @@ -49,4 +51,5 @@ public class EdgeEvent extends BaseData { public EdgeEvent(EdgeEventId id) { super(id); } + } diff --git a/common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeGrpcClient.java b/common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeGrpcClient.java index 1d3b157ec2..fcef33b968 100644 --- a/common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeGrpcClient.java +++ b/common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeGrpcClient.java @@ -136,7 +136,7 @@ public class EdgeGrpcClient implements EdgeRpcClient { .setConnectRequestMsg(ConnectRequestMsg.newBuilder() .setEdgeRoutingKey(edgeKey) .setEdgeSecret(edgeSecret) - .setEdgeVersion(EdgeVersion.V_3_8_0) + .setEdgeVersion(EdgeVersion.V_3_9_0) .setMaxInboundMessageSize(maxInboundMessageSize) .build()) .build()); diff --git a/common/edge-api/src/main/proto/edge.proto b/common/edge-api/src/main/proto/edge.proto index 5fb51de2ab..04f7c2031d 100644 --- a/common/edge-api/src/main/proto/edge.proto +++ b/common/edge-api/src/main/proto/edge.proto @@ -40,6 +40,7 @@ enum EdgeVersion { V_3_6_4 = 6; V_3_7_0 = 7; V_3_8_0 = 8; + V_3_9_0 = 9; } /** @@ -635,6 +636,10 @@ message ProcessingStrategyProto { int64 maxPauseBetweenRetries = 5; } +message EdgeEvent { + +} + /** * Main Messages; */ diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/edge/EdgeEventUpdateMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/edge/EdgeEventUpdateMsg.java index 892b2a57dd..4c7ccb58ec 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/edge/EdgeEventUpdateMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/edge/EdgeEventUpdateMsg.java @@ -20,9 +20,12 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.MsgType; +import java.io.Serial; + @Data public class EdgeEventUpdateMsg implements EdgeSessionMsg { + @Serial private static final long serialVersionUID = -8050114506822836537L; private final TenantId tenantId; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/edge/EdgeHighPriorityMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/edge/EdgeHighPriorityMsg.java index 03f039f2e4..325ae828e9 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/edge/EdgeHighPriorityMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/edge/EdgeHighPriorityMsg.java @@ -20,9 +20,14 @@ import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.MsgType; +import java.io.Serial; + @Data public class EdgeHighPriorityMsg implements EdgeSessionMsg { + @Serial + private static final long serialVersionUID = 2703437686242033551L; + private final TenantId tenantId; private final EdgeEvent edgeEvent; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/edge/EdgeSessionMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/edge/EdgeSessionMsg.java index 4e658d6dd4..491deac52b 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/edge/EdgeSessionMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/edge/EdgeSessionMsg.java @@ -18,7 +18,9 @@ package org.thingsboard.server.common.msg.edge; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.MsgType; -public interface EdgeSessionMsg { +import java.io.Serializable; + +public interface EdgeSessionMsg extends Serializable { TenantId getTenantId(); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/edge/FromEdgeSyncResponse.java b/common/message/src/main/java/org/thingsboard/server/common/msg/edge/FromEdgeSyncResponse.java index 1e8beea443..d6bb5c0e7f 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/edge/FromEdgeSyncResponse.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/edge/FromEdgeSyncResponse.java @@ -20,11 +20,13 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.MsgType; +import java.io.Serial; import java.util.UUID; @Data public class FromEdgeSyncResponse implements EdgeSessionMsg { + @Serial private static final long serialVersionUID = -6360890556315667486L; private final UUID id; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/edge/ToEdgeSyncRequest.java b/common/message/src/main/java/org/thingsboard/server/common/msg/edge/ToEdgeSyncRequest.java index 04516b58c5..97ace1ccb1 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/edge/ToEdgeSyncRequest.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/edge/ToEdgeSyncRequest.java @@ -20,11 +20,13 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.MsgType; +import java.io.Serial; import java.util.UUID; @Data public class ToEdgeSyncRequest implements EdgeSessionMsg { + @Serial private static final long serialVersionUID = -7624597032448212259L; private final UUID id; diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index ab7a50705e..405371d27a 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.device.data.CoapDeviceTransportConfigu import org.thingsboard.server.common.data.device.data.Lwm2mDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.PowerMode; import org.thingsboard.server.common.data.device.data.PowerSavingConfiguration; +import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.ApiUsageStateId; @@ -181,6 +182,43 @@ public class ProtoUtils { ); } + public static TransportProtos.EdgeEventMsgProto toProto(EdgeEvent edgeEvent) { + TransportProtos.EdgeEventMsgProto.Builder builder = TransportProtos.EdgeEventMsgProto.newBuilder(); + + builder.setTenantIdMSB(edgeEvent.getTenantId().getId().getMostSignificantBits()); + builder.setTenantIdLSB(edgeEvent.getTenantId().getId().getLeastSignificantBits()); + builder.setEntityType(edgeEvent.getType().name()); + builder.setAction(edgeEvent.getAction().name()); + + if (edgeEvent.getEntityId() != null) { + builder.setEntityIdMSB(edgeEvent.getEntityId().getMostSignificantBits()); + builder.setEntityIdLSB(edgeEvent.getEntityId().getLeastSignificantBits()); + } + if (edgeEvent.getBody() != null) { + builder.setBody(JacksonUtil.toString(edgeEvent.getBody())); + } + + return builder.build(); + } + + public static EdgeEvent fromProto(TransportProtos.EdgeEventMsgProto proto) { + EdgeEvent edgeEvent = new EdgeEvent(); + TenantId tenantId = new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + edgeEvent.setTenantId(tenantId); + edgeEvent.setType(EdgeEventType.valueOf(proto.getEntityType())); + edgeEvent.setAction(EdgeEventActionType.valueOf(proto.getAction())); + + if (proto.hasEntityIdMSB() && proto.hasEntityIdLSB()) { + edgeEvent.setEntityId(new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + } + + if (proto.hasBody()) { + edgeEvent.setBody(JacksonUtil.toJsonNode(proto.getBody())); + } + + return edgeEvent; + } + public static TransportProtos.EdgeHighPriorityMsgProto toProto(EdgeHighPriorityMsg msg) { TransportProtos.EdgeHighPriorityMsgProto.Builder builder = TransportProtos.EdgeHighPriorityMsgProto.newBuilder() .setTenantIdMSB(msg.getTenantId().getId().getMostSignificantBits()) diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index f84c7f7c9b..5d3ce2a272 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1126,6 +1126,16 @@ message ComponentLifecycleMsgProto { ComponentLifecycleEvent event = 6; } +message EdgeEventMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + string entityType = 3; + string action = 4; + optional int64 entityIdMSB = 5; + optional int64 entityIdLSB = 6; + optional string body = 7; +} + message EdgeNotificationMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; @@ -1521,7 +1531,6 @@ message ToCoreNotificationMsg { RestApiCallResponseMsgProto restApiCallResponseMsg = 50; } - /* Messages to Edge queue that are handled by ThingsBoard Core Service */ message ToEdgeMsg { EdgeNotificationMsgProto edgeNotificationMsg = 1; @@ -1535,6 +1544,10 @@ message ToEdgeNotificationMsg { ComponentLifecycleMsgProto componentLifecycle = 5; } +message ToEdgeEventNotificationMsg { + EdgeEventMsgProto edgeEventMsg = 1; +} + /* Messages that are handled by ThingsBoard RuleEngine Service */ message ToRuleEngineMsg { int64 tenantIdMSB = 1; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java index 8bd15a1ccf..f45c003c0c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.discovery; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -33,6 +34,7 @@ public class TopicService { private final ConcurrentMap tbCoreNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentMap tbRuleEngineNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentMap tbEdgeNotificationTopics = new ConcurrentHashMap<>(); + private final ConcurrentMap tbEdgeEventsNotificationTopics = new ConcurrentHashMap<>(); /** * Each Service should start a consumer for messages that target individual service instance based on serviceId. @@ -59,6 +61,14 @@ public class TopicService { return buildTopicPartitionInfo("tb_edge.notifications." + serviceId, null, null, false); } + public TopicPartitionInfo getEdgeEventNotificationsTopic(TenantId tenantId, EdgeId edgeId, String serviceId) { + return tbEdgeEventsNotificationTopics.computeIfAbsent(edgeId, id -> buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId, serviceId)); + } + + private TopicPartitionInfo buildEdgeEventNotificationsTopicPartitionInfo(TenantId tenantId, EdgeId edgeId, String serviceId) { + return buildTopicPartitionInfo("tb_edge_event.notifications." + tenantId + "." + edgeId + "." + serviceId, null, null, false); + } + private TopicPartitionInfo buildNotificationsTopicPartitionInfo(ServiceType serviceType, String serviceId) { return buildTopicPartitionInfo(serviceType.name().toLowerCase() + ".notifications." + serviceId, null, null, false); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 005e9aee17..6c758409d8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -20,12 +20,15 @@ import jakarta.annotation.PreDestroy; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -281,6 +284,16 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); } + @Override + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + return null; + } + + @Override + public TbQueueProducer> createEdgeEventMsgProducer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 515c1bba44..4f9c130986 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -20,10 +20,13 @@ import jakarta.annotation.PreDestroy; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -261,6 +264,16 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); } + @Override + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + return null; + } + + @Override + public TbQueueProducer> createEdgeEventMsgProducer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index 37319e378b..04ca87eabb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -23,10 +23,11 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -131,10 +132,15 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory } @Override - public TbQueueProducer> createEdgeNotificationsMsgProducer() { + public TbQueueProducer> createEdgeNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); } + @Override + public TbQueueProducer> createEdgeEventMsgProducer() { + return null; + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, topicService.buildTopicName(configuration.getTopic()), diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index c4281bc390..b3c2a2f44c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -19,6 +19,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -204,6 +206,16 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new InMemoryTbQueueProducer<>(storage, topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); } + @Override + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + return null; + } + + @Override + public TbQueueProducer> createEdgeEventMsgProducer() { + return null; + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 46d94f094f..0c5cc94a72 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -20,11 +20,14 @@ import jakarta.annotation.PreDestroy; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -462,6 +465,29 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName("tb_edge_event.notifications." + tenantId + "." + edgeId + "." + serviceId)); + consumerBuilder.clientId("monolith-to-edge-event-consumer" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId(topicService.buildTopicName("monolith-edge-event-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeEventNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(edgeAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createEdgeEventMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-to-edge-event-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName("edge-events")); + requestBuilder.admin(edgeAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index b8b51706b2..0866f1cde7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -20,10 +20,13 @@ import jakarta.annotation.PreDestroy; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -411,6 +414,29 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName("tb_edge_event.notifications." + tenantId + "." + edgeId + "." + serviceId)); + consumerBuilder.clientId("tb-core-edge-event-consumer" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId(topicService.buildTopicName("tb-core-edge-event-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeEventNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(edgeAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createEdgeEventMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-edge-event-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName("edge-events")); + requestBuilder.admin(edgeAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 485830caf4..7090b87a66 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -23,10 +23,11 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -181,8 +182,8 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { } @Override - public TbQueueProducer> createEdgeNotificationsMsgProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + public TbQueueProducer> createEdgeNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-edge-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); @@ -190,6 +191,16 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createEdgeEventMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-edge-event-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName("edge-events")); + requestBuilder.admin(edgeAdmin); + return requestBuilder.build(); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { throw new UnsupportedOperationException("Rule engine consumer should use a partitionId"); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index 3df48e8fb8..7859e4f6fa 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -20,12 +20,15 @@ import jakarta.annotation.PreDestroy; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -282,6 +285,16 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); } + @Override + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + return null; + } + + @Override + public TbQueueProducer> createEdgeEventMsgProducer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 919d895a97..3a0db39079 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -20,10 +20,13 @@ import jakarta.annotation.PreDestroy; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -254,6 +257,16 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); } + @Override + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + return null; + } + + @Override + public TbQueueProducer> createEdgeEventMsgProducer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index bb46610de5..671da15f20 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -23,10 +23,11 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -130,10 +131,15 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory } @Override - public TbQueueProducer> createEdgeNotificationsMsgProducer() { + public TbQueueProducer> createEdgeNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); } + @Override + public TbQueueProducer> createEdgeEventMsgProducer() { + return null; + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, topicService.buildTopicName(configuration.getTopic()), diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index 8ad400d9e3..d6ec066db8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -20,12 +20,15 @@ import jakarta.annotation.PreDestroy; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -280,6 +283,16 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); } + @Override + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + return null; + } + + @Override + public TbQueueProducer> createEdgeEventMsgProducer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index 16f9838384..0896774c33 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -20,10 +20,13 @@ import jakarta.annotation.PreDestroy; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -170,6 +173,16 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); } + @Override + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + return null; + } + + @Override + public TbQueueProducer> createEdgeEventMsgProducer() { + return null; + } + @Override public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, topicService.buildTopicName(transportApiSettings.getRequestsTopic()), diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java index 3dabaf100b..2f6a5c9eb6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java @@ -23,10 +23,11 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -129,10 +130,15 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor } @Override - public TbQueueProducer> createEdgeNotificationsMsgProducer() { + public TbQueueProducer> createEdgeNotificationsMsgProducer() { return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); } + @Override + public TbQueueProducer> createEdgeEventMsgProducer() { + return null; + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, topicService.buildTopicName(configuration.getTopic()), diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index fa7e73b879..c686d713f2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -20,11 +20,14 @@ import jakarta.annotation.PreDestroy; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -279,6 +282,16 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); } + @Override + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + return null; + } + + @Override + public TbQueueProducer> createEdgeEventMsgProducer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index 71a7efe50b..09abff3ece 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -20,10 +20,13 @@ import jakarta.annotation.PreDestroy; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -254,6 +257,16 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); } + @Override + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + return null; + } + + @Override + public TbQueueProducer> createEdgeEventMsgProducer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java index 643a9dee3b..916c778722 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java @@ -23,10 +23,11 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -129,10 +130,15 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact } @Override - public TbQueueProducer> createEdgeNotificationsMsgProducer() { + public TbQueueProducer> createEdgeNotificationsMsgProducer() { return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); } + @Override + public TbQueueProducer> createEdgeEventMsgProducer() { + return null; + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new TbServiceBusConsumerTemplate<>(ruleEngineAdmin, serviceBusSettings, topicService.buildTopicName(configuration.getTopic()), diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index cecfddf7a7..63fdce09e5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -15,9 +15,12 @@ */ package org.thingsboard.server.queue.provider; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -148,4 +151,8 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, Hous TbQueueProducer> createEdgeNotificationsMsgProducer(); + TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId); + + TbQueueProducer> createEdgeEventMsgProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index cf94bf68d0..9cf18e6cb4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -19,6 +19,7 @@ import jakarta.annotation.PostConstruct; import org.springframework.stereotype.Service; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -43,6 +44,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toTbCoreNotifications; private TbQueueProducer> toEdge; private TbQueueProducer> toEdgeNotifications; + private TbQueueProducer> toEdgeEvents; private TbQueueProducer> toUsageStats; private TbQueueProducer> toVersionControl; private TbQueueProducer> toHousekeeper; @@ -63,6 +65,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toHousekeeper = tbQueueProvider.createHousekeeperMsgProducer(); this.toEdge = tbQueueProvider.createEdgeMsgProducer(); this.toEdgeNotifications = tbQueueProvider.createEdgeNotificationsMsgProducer(); + this.toEdgeEvents = tbQueueProvider.createEdgeEventMsgProducer(); } @Override @@ -116,4 +119,9 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { return toEdgeNotifications; } + @Override + public TbQueueProducer> getTbEdgeEventsMsgProducer() { + return toEdgeEvents; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index a0c43a9e4b..ec31763baa 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -88,4 +89,6 @@ public interface TbQueueProducerProvider { TbQueueProducer> getTbEdgeNotificationsMsgProducer(); + TbQueueProducer> getTbEdgeEventsMsgProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index 8797064204..e9f7773a26 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -20,6 +20,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -45,6 +46,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toHousekeeper; private TbQueueProducer> toEdge; private TbQueueProducer> toEdgeNotifications; + private TbQueueProducer> toEdgeEvents; public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -61,6 +63,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { this.toHousekeeper = tbQueueProvider.createHousekeeperMsgProducer(); this.toEdge = tbQueueProvider.createEdgeMsgProducer(); this.toEdgeNotifications = tbQueueProvider.createEdgeNotificationsMsgProducer(); + this.toEdgeEvents = tbQueueProvider.createEdgeEventMsgProducer(); } @Override @@ -98,6 +101,11 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { return toEdgeNotifications; } + @Override + public TbQueueProducer> getTbEdgeEventsMsgProducer() { + return toEdgeEvents; + } + @Override public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index dda3f33ed4..a25b590b9a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -17,10 +17,11 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -74,7 +75,9 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory TbQueueProducer> createEdgeMsgProducer(); - TbQueueProducer> createEdgeNotificationsMsgProducer(); + TbQueueProducer> createEdgeNotificationsMsgProducer(); + + TbQueueProducer> createEdgeEventMsgProducer(); /** * Used to consume messages about firmware update notifications to TB Core Service diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index b2c7856b20..2540cdeafa 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -20,6 +20,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -90,6 +91,11 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } + @Override + public TbQueueProducer> getTbEdgeEventsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used Transport!"); + } + @Override public TbQueueProducer> getTbVersionControlMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Transport!"); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java index a8f3b124e2..cd4fa12df0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -20,6 +20,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -86,6 +87,11 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); } + @Override + public TbQueueProducer> getTbEdgeEventsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + @Override public TbQueueProducer> getTbVersionControlMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/BaseEdgeEventService.java b/dao/src/main/java/org/thingsboard/server/dao/edge/BaseEdgeEventService.java index 274b0154a5..e7780d80ad 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/BaseEdgeEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/BaseEdgeEventService.java @@ -23,6 +23,7 @@ import jakarta.annotation.PreDestroy; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; @@ -41,15 +42,15 @@ import org.thingsboard.server.dao.service.DataValidator; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -@Service @Slf4j +@Service @RequiredArgsConstructor +@ConditionalOnExpression("'${queue.type:null}'!='kafka'") public class BaseEdgeEventService implements EdgeEventService { private final EdgeEventDao edgeEventDao; private final RateLimitService rateLimitService; private final DataValidator edgeEventValidator; - private final ApplicationEventPublisher eventPublisher; private ExecutorService edgeEventExecutor; @@ -101,4 +102,5 @@ public class BaseEdgeEventService implements EdgeEventService { public void cleanupEvents(long ttl) { edgeEventDao.cleanupEvents(ttl); } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/DefaultEdgeSynchronizationManager.java b/dao/src/main/java/org/thingsboard/server/dao/edge/DefaultEdgeSynchronizationManager.java index 1fc9bdb226..c1016c571d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/DefaultEdgeSynchronizationManager.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/DefaultEdgeSynchronizationManager.java @@ -20,10 +20,11 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.EdgeId; -@Component @Slf4j +@Getter +@Component public class DefaultEdgeSynchronizationManager implements EdgeSynchronizationManager { - @Getter private final ThreadLocal edgeId = new ThreadLocal<>(); + } From 9ba71056cebff1976d92e032abe15c35d46b704a Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 25 Oct 2024 11:52:54 +0300 Subject: [PATCH 16/71] refactored UserService to reuse functionality --- .../service/entitiy/user/DefaultUserService.java | 10 ++-------- .../org/thingsboard/server/dao/user/UserService.java | 2 ++ .../thingsboard/server/dao/user/UserServiceImpl.java | 10 ++++++++++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/user/DefaultUserService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/user/DefaultUserService.java index ad4b7c097e..3fe8f27873 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/user/DefaultUserService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/user/DefaultUserService.java @@ -90,16 +90,10 @@ public class DefaultUserService extends AbstractTbEntityService implements TbUse public UserActivationLink getActivationLink(TenantId tenantId, CustomerId customerId, UserId userId, HttpServletRequest request) throws ThingsboardException { UserCredentials userCredentials = userService.findUserCredentialsByUserId(tenantId, userId); if (!userCredentials.isEnabled() && userCredentials.getActivateToken() != null) { - long ttl = userCredentials.getActivationTokenTtl(); - if (ttl < TimeUnit.MINUTES.toMillis(15)) { // renew link if less than 15 minutes before expiration - userCredentials = userService.generateUserActivationToken(userCredentials); - userCredentials = userService.saveUserCredentials(tenantId, userCredentials); - ttl = userCredentials.getActivationTokenTtl(); - log.debug("[{}][{}] Regenerated expired user activation token", tenantId, userId); - } + userCredentials = userService.refreshUserActivationToken(tenantId, userCredentials); String baseUrl = systemSecurityService.getBaseUrl(tenantId, customerId, request); String link = baseUrl + "/api/noauth/activate?activateToken=" + userCredentials.getActivateToken(); - return new UserActivationLink(link, ttl); + return new UserActivationLink(link, userCredentials.getActivationTokenTtl()); } else { throw new ThingsboardException("User is already activated!", ThingsboardErrorCode.BAD_REQUEST_PARAMS); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java index 8f22812cfc..0d532cdfac 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java @@ -63,6 +63,8 @@ public interface UserService extends EntityDaoService { UserCredentials generateUserActivationToken(UserCredentials userCredentials); + UserCredentials refreshUserActivationToken(TenantId tenantId, UserCredentials userCredentials); + UserCredentials replaceUserCredentials(TenantId tenantId, UserCredentials userCredentials); void deleteUser(TenantId tenantId, User user); diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index ba0d3d69f3..d8eee2967a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -300,6 +300,16 @@ public class UserServiceImpl extends AbstractCachedEntityService Date: Wed, 30 Oct 2024 12:29:07 +0200 Subject: [PATCH 17/71] Implement clean up service to clear empty edge topics --- .../edge/EdgeEventSourcingListener.java | 25 +++- .../edge/rpc/AbstractEdgeGrpcSession.java | 30 ++--- .../service/edge/rpc/EdgeGrpcService.java | 4 +- .../edge/rpc/KafkaEdgeEventService.java | 16 +-- .../edge/rpc/KafkaEdgeGrpcSession.java | 19 ++- .../edge/rpc/PostgresEdgeGrpcSession.java | 9 ++ .../edge/rpc/processor/BaseEdgeProcessor.java | 8 -- .../service/ttl/AbstractCleanUpService.java | 3 +- .../service/ttl/EdgeEventsCleanUpService.java | 2 +- .../ttl/KafkaEdgeTopicsCleanUpService.java | 111 ++++++++++++++++++ .../src/main/resources/thingsboard.yml | 3 + common/edge-api/src/main/proto/edge.proto | 4 - .../server/queue/discovery/TopicService.java | 8 +- .../server/queue/kafka/TbKafkaAdmin.java | 9 ++ .../queue/kafka/TbKafkaTopicConfigs.java | 5 + .../provider/AwsSqsMonolithQueueFactory.java | 2 +- .../provider/AwsSqsTbCoreQueueFactory.java | 2 +- .../InMemoryMonolithQueueFactory.java | 2 +- .../provider/KafkaMonolithQueueFactory.java | 10 +- .../provider/KafkaTbCoreQueueFactory.java | 10 +- .../KafkaTbRuleEngineQueueFactory.java | 4 +- .../provider/PubSubMonolithQueueFactory.java | 2 +- .../provider/PubSubTbCoreQueueFactory.java | 2 +- .../RabbitMqMonolithQueueFactory.java | 2 +- .../provider/RabbitMqTbCoreQueueFactory.java | 2 +- .../ServiceBusMonolithQueueFactory.java | 2 +- .../ServiceBusTbCoreQueueFactory.java | 2 +- .../queue/provider/TbCoreQueueFactory.java | 2 +- .../server/dao/edge/EdgeServiceImpl.java | 1 + 29 files changed, 228 insertions(+), 73 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index 893bd13171..7706244680 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -20,6 +20,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.common.util.JacksonUtil; @@ -37,6 +39,7 @@ import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; +import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -50,6 +53,8 @@ import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.discovery.TopicService; /** * This event listener does not support async event processing because relay on ThreadLocal @@ -65,14 +70,20 @@ import org.thingsboard.server.dao.tenant.TenantService; * future.addCallback(eventPublisher.publishEvent(...)) * } * */ +@Slf4j @Component @RequiredArgsConstructor -@Slf4j +@ConditionalOnExpression("${edges.enabled:true}") public class EdgeEventSourcingListener { + private final TopicService topicService; + private final TbQueueAdmin tbQueueAdmin; + private final TenantService tenantService; private final TbClusterService tbClusterService; private final EdgeSynchronizationManager edgeSynchronizationManager; - private final TenantService tenantService; + + @Value("#{'${queue.type:null}' == 'kafka'}") + private boolean isKafkaSupported; @PostConstruct public void init() { @@ -106,7 +117,15 @@ public class EdgeEventSourcingListener { return; } try { - if (EntityType.EDGE.equals(entityType) || EntityType.TENANT.equals(entityType)) { + if (EntityType.EDGE.equals(entityType)) { + if (isKafkaSupported) { + String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, (EdgeId) event.getEntityId()).getTopic(); + tbQueueAdmin.deleteTopic(topic); + } else { + return; + } + } + if (EntityType.TENANT.equals(entityType)) { return; } log.trace("[{}] DeleteEntityEvent called: {}", tenantId, event); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java index f92daa588a..397e1f6137 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java @@ -102,40 +102,34 @@ import java.util.function.BiConsumer; @Data public abstract class AbstractEdgeGrpcSession> implements EdgeGrpcSession, Closeable { - protected static final ReentrantLock downlinkMsgLock = new ReentrantLock(); - protected static final ConcurrentLinkedQueue highPriorityQueue = new ConcurrentLinkedQueue<>(); + private static final int MAX_DOWNLINK_ATTEMPTS = 10; + private static final String RATE_LIMIT_REACHED = "Rate limit reached"; - protected static final int MAX_DOWNLINK_ATTEMPTS = 10; - protected static final String QUEUE_START_TS_ATTR_KEY = "queueStartTs"; - protected static final String QUEUE_START_SEQ_ID_ATTR_KEY = "queueStartSeqId"; - protected static final String RATE_LIMIT_REACHED = "Rate limit reached"; + protected static final ConcurrentLinkedQueue highPriorityQueue = new ConcurrentLinkedQueue<>(); protected UUID sessionId; protected BiConsumer sessionOpenListener; protected BiConsumer sessionCloseListener; - protected final EdgeSessionState sessionState = new EdgeSessionState(); + private final EdgeSessionState sessionState = new EdgeSessionState(); + private final ReentrantLock downlinkMsgLock = new ReentrantLock(); protected EdgeContextComponent ctx; protected Edge edge; protected TenantId tenantId; + protected StreamObserver inputStream; protected StreamObserver outputStream; + protected boolean connected; protected volatile boolean syncCompleted; - protected Long newStartTs; - protected Long previousStartTs; - protected Long newStartSeqId; - protected Long previousStartSeqId; - protected Long seqIdEnd; - - protected EdgeVersion edgeVersion; - protected int maxInboundMessageSize; - protected int clientMaxInboundMessageSize; - protected int maxHighPriorityQueueSizePerSession; + private EdgeVersion edgeVersion; + private int maxInboundMessageSize; + private int clientMaxInboundMessageSize; + private int maxHighPriorityQueueSizePerSession; - protected ScheduledExecutorService sendDownlinkExecutorService; + private ScheduledExecutorService sendDownlinkExecutorService; public AbstractEdgeGrpcSession(EdgeContextComponent ctx, StreamObserver outputStream, BiConsumer sessionOpenListener, diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java index 53368dfc06..bc63eb251a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java @@ -122,7 +122,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i @Value("${edges.max_high_priority_queue_size_per_session:10000}") private int maxHighPriorityQueueSizePerSession; - @Value("#{ '${queue.type:null}' == 'kafka' }") + @Value("#{'${queue.type:null}' == 'kafka'}") private boolean isKafkaSupported; @Autowired @@ -335,7 +335,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i save(tenantId, edgeId, DefaultDeviceStateService.LAST_CONNECT_TIME, lastConnectTs); edgeIdServiceIdCache.put(edgeId, serviceInfoProvider.getServiceId()); if (isKafkaSupported) { - TbQueueConsumer> consumer = tbCoreQueueFactory.createEdgeEventMsgConsumer(tenantId, edgeId, serviceInfoProvider.getServiceId()); + TbQueueConsumer> consumer = tbCoreQueueFactory.createEdgeEventMsgConsumer(tenantId, edgeId); ((KafkaEdgeGrpcSession) edgeGrpcSession).initConsumer(consumer); } pushRuleEngineMessage(tenantId, edge, lastConnectTs, TbMsgType.CONNECT_EVENT); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java index 517b550a36..3caea5476a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -23,7 +23,6 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; -import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.cache.limits.RateLimitService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.edge.EdgeEvent; @@ -38,12 +37,11 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeEventService; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import java.util.Optional; import java.util.UUID; @Slf4j @@ -59,7 +57,6 @@ public class KafkaEdgeEventService implements EdgeEventService { private final TbQueueProducerProvider producerProvider; @Lazy private final TopicService topicService; - private final TbTransactionalCache edgeIdServiceIdCache; @Override public ListenableFuture saveAsync(EdgeEvent edgeEvent) { @@ -70,9 +67,8 @@ public class KafkaEdgeEventService implements EdgeEventService { throw new TbRateLimitsException(EntityType.EDGE); } edgeEventValidator.validate(edgeEvent, EdgeEvent::getTenantId); - var serviceIdOpt = Optional.ofNullable(edgeIdServiceIdCache.get(edgeEvent.getEdgeId())); - TopicPartitionInfo tpi = topicService.getEdgeEventNotificationsTopic(edgeEvent.getTenantId(), edgeEvent.getEdgeId(), serviceIdOpt.get().get()); - TransportProtos.ToEdgeEventNotificationMsg msg = TransportProtos.ToEdgeEventNotificationMsg.newBuilder().setEdgeEventMsg(ProtoUtils.toProto(edgeEvent)).build(); + TopicPartitionInfo tpi = topicService.getEdgeEventNotificationsTopic(edgeEvent.getTenantId(), edgeEvent.getEdgeId()); + ToEdgeEventNotificationMsg msg = ToEdgeEventNotificationMsg.newBuilder().setEdgeEventMsg(ProtoUtils.toProto(edgeEvent)).build(); producerProvider.getTbEdgeEventsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(edgeEvent.getTenantId()).entity(edgeEvent).entityId(edgeEvent.getEdgeId()).build()); @@ -85,8 +81,6 @@ public class KafkaEdgeEventService implements EdgeEventService { } @Override - public void cleanupEvents(long ttl) { - - } + public void cleanupEvents(long ttl) {} } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java index 78acd02d24..b7857d4e7d 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.service.edge.rpc; +import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import io.grpc.stub.StreamObserver; import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.EdgeId; @@ -83,8 +85,21 @@ public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession downlinkMsgsPack = convertToDownlinkMsgsPack(edgeEvents); - sendDownlinkMsgsPack(downlinkMsgsPack).get(); - edgeEventsConsumer.commit(); + Futures.addCallback(sendDownlinkMsgsPack(downlinkMsgsPack), new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Boolean isInterrupted) { + if (Boolean.TRUE.equals(isInterrupted)) { + log.debug("[{}][{}][{}] Send downlink messages task was interrupted", tenantId, edge.getId(), sessionId); + } else { + edgeEventsConsumer.commit(); + processEdgeEvents(); + } + } + @Override + public void onFailure(Throwable t) { + log.error("[{}] Failed to send downlink msgs pack", sessionId, t); + } + }, ctx.getGrpcCallbackExecutorService()); return Futures.immediateFuture(Boolean.TRUE); } catch (Exception e) { log.error("[{}][{}] Error occurred while polling edge events from Kafka: {}", tenantId, edge.getId(), e.getMessage()); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java index 47fad107a1..8293942523 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java @@ -46,6 +46,15 @@ import java.util.function.BiConsumer; @Slf4j public class PostgresEdgeGrpcSession extends AbstractEdgeGrpcSession { + private static final String QUEUE_START_TS_ATTR_KEY = "queueStartTs"; + private static final String QUEUE_START_SEQ_ID_ATTR_KEY = "queueStartSeqId"; + + private Long newStartTs; + private Long previousStartTs; + private Long newStartSeqId; + private Long previousStartSeqId; + private Long seqIdEnd; + PostgresEdgeGrpcSession(EdgeContextComponent ctx, StreamObserver outputStream, BiConsumer sessionOpenListener, BiConsumer sessionCloseListener, ScheduledExecutorService sendDownlinkExecutorService, diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index 7c823a0dfa..eb6b0bf47a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -20,9 +20,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.EntityType; @@ -57,9 +55,6 @@ import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.edge.EdgeContextComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.state.DefaultDeviceStateService; @@ -88,9 +83,6 @@ public abstract class BaseEdgeProcessor { @Autowired protected DbCallbackExecutorService dbCallbackExecutorService; - @Autowired - private TbTransactionalCache edgeIdServiceIdCache; - protected ListenableFuture saveEdgeEvent(TenantId tenantId, EdgeId edgeId, EdgeEventType type, diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java index f435be07ff..3eb8348fae 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java @@ -28,7 +28,8 @@ public abstract class AbstractCleanUpService { private final PartitionService partitionService; - protected boolean isSystemTenantPartitionMine(){ + protected boolean isSystemTenantPartitionMine() { return partitionService.resolve(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID).isMyPartition(); } + } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java index 7c025d8703..ff4d574ea4 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java @@ -32,7 +32,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.EDGE_EVENT_TABLE_N @TbCoreComponent @Slf4j @Service -@ConditionalOnExpression("${sql.ttl.edge_events.enabled:true} && ${sql.ttl.edge_events.edge_events_ttl:0} > 0") +@ConditionalOnExpression("${edges.enabled:true} && ${sql.ttl.edge_events.edge_events_ttl:0} > 0") public class EdgeEventsCleanUpService extends AbstractCleanUpService { public static final String RANDOM_DELAY_INTERVAL_MS_EXPRESSION = diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java new file mode 100644 index 0000000000..a2d93cdd6b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java @@ -0,0 +1,111 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.state.DefaultDeviceStateService; + +import java.time.Instant; +import java.util.Date; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +@Slf4j +@Service +@TbCoreComponent +@RequiredArgsConstructor +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && ${edges.enabled:true}") +public class KafkaEdgeTopicsCleanUpService { + + private static final long ONE_MONTH_MILLIS = TimeUnit.DAYS.toChronoUnit().getDuration().multipliedBy(30).toMillis(); + + private final EdgeService edgeService; + private final TenantService tenantService; + private final AttributesService attributesService; + + private final TopicService topicService; + private final PartitionService partitionService; + + private final TbKafkaSettings kafkaSettings; + private final TbKafkaTopicConfigs kafkaTopicConfigs; + + private final ExecutorService executorService = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("kafka-edge-topic-cleanup")); + + @Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.edge_events.checking_interval})}", fixedDelayString = "${sql.ttl.edge_events.checking_interval}") + public void cleanUp() { + executorService.submit(() -> { + PageDataIterable tenants = new PageDataIterable<>(tenantService::findTenantsIds, 10_000); + for (TenantId tenantId : tenants) { + try { + cleanUp(tenantId); + } catch (Exception e) { + log.warn("Failed to drop kafka topics for tenant {}", tenantId, e); + } + } + }); + } + + private void cleanUp(TenantId tenantId) throws Exception { + if (!partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) { + return; + } + + PageDataIterable edges = new PageDataIterable<>(link -> edgeService.findEdgesByTenantId(tenantId, link), 1024); + long currentTimeMillis = System.currentTimeMillis(); + + for (Edge edge : edges) { + Optional attributeOpt = attributesService.find(tenantId, edge.getId(), AttributeScope.SERVER_SCOPE, DefaultDeviceStateService.LAST_CONNECT_TIME).get(); + if (attributeOpt.isPresent()) { + Optional lastConnectTimeOpt = attributeOpt.get().getLongValue(); + if (lastConnectTimeOpt.isPresent() && isTopicExpired(lastConnectTimeOpt.get(), currentTimeMillis)) { + String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edge.getId()).getTopic(); + TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + if (kafkaAdmin.isTopicEmpty(topic)) { + kafkaAdmin.deleteTopic(topic); + log.info("Removed outdated topic for tenant {} and edge {} older than {}", tenantId, edge.getName(), Date.from(Instant.ofEpochMilli(currentTimeMillis - ONE_MONTH_MILLIS))); + } + } + } + } + } + + private boolean isTopicExpired(long lastConnectTime, long currentTimeMillis) { + return lastConnectTime + ONE_MONTH_MILLIS < currentTimeMillis; + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 9a7994e92a..180a978744 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1554,6 +1554,9 @@ queue: housekeeper-reprocessing: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_TOPIC_PROPERTIES:retention.ms:7776000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Edge topic edge: "${TB_QUEUE_KAFKA_EDGE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Edge event topic + # TODO : discuss and adjust properties + edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:2592000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" diff --git a/common/edge-api/src/main/proto/edge.proto b/common/edge-api/src/main/proto/edge.proto index 04f7c2031d..11d320ffa7 100644 --- a/common/edge-api/src/main/proto/edge.proto +++ b/common/edge-api/src/main/proto/edge.proto @@ -636,10 +636,6 @@ message ProcessingStrategyProto { int64 maxPauseBetweenRetries = 5; } -message EdgeEvent { - -} - /** * Main Messages; */ diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java index f45c003c0c..168e90cd1b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java @@ -61,12 +61,12 @@ public class TopicService { return buildTopicPartitionInfo("tb_edge.notifications." + serviceId, null, null, false); } - public TopicPartitionInfo getEdgeEventNotificationsTopic(TenantId tenantId, EdgeId edgeId, String serviceId) { - return tbEdgeEventsNotificationTopics.computeIfAbsent(edgeId, id -> buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId, serviceId)); + public TopicPartitionInfo getEdgeEventNotificationsTopic(TenantId tenantId, EdgeId edgeId) { + return tbEdgeEventsNotificationTopics.computeIfAbsent(edgeId, id -> buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId)); } - private TopicPartitionInfo buildEdgeEventNotificationsTopicPartitionInfo(TenantId tenantId, EdgeId edgeId, String serviceId) { - return buildTopicPartitionInfo("tb_edge_event.notifications." + tenantId + "." + edgeId + "." + serviceId, null, null, false); + public TopicPartitionInfo buildEdgeEventNotificationsTopicPartitionInfo(TenantId tenantId, EdgeId edgeId) { + return buildTopicPartitionInfo("tb_edge_event.notifications." + tenantId + "." + edgeId, null, null, false); } private TopicPartitionInfo buildNotificationsTopicPartitionInfo(ServiceType serviceType, String serviceId) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index 2bd283f1e5..d614fe4fdd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -170,7 +170,16 @@ public class TbKafkaAdmin implements TbQueueAdmin { log.info("[{}] altered new consumer groupId {}", tp, newGroupId); break; } + } + public boolean isTopicEmpty(String topic) { + try { + Map offsets = settings.getAdminClient().listConsumerGroupOffsets("__consumer_offsets").partitionsToOffsetAndMetadata().get(); + return offsets.entrySet().stream().noneMatch(entry -> entry.getKey().topic().equals(topic)); + } catch (InterruptedException | ExecutionException e) { + log.error("Failed to check if topic [{}] is empty.", topic, e); + return false; + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index 300548e081..ee529e8a68 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -46,6 +46,8 @@ public class TbKafkaTopicConfigs { private String vcProperties; @Value("${queue.kafka.topic-properties.edge:}") private String edgeProperties; + @Value("${queue.kafka.topic-properties.edge-event:}") + private String edgeEventProperties; @Value("${queue.kafka.topic-properties.housekeeper:}") private String housekeeperProperties; @Value("${queue.kafka.topic-properties.housekeeper-reprocessing:}") @@ -75,6 +77,8 @@ public class TbKafkaTopicConfigs { private Map housekeeperReprocessingConfigs; @Getter private Map edgeConfigs; + @Getter + private Map edgeEventConfigs; @PostConstruct private void init() { @@ -92,6 +96,7 @@ public class TbKafkaTopicConfigs { housekeeperConfigs = PropertyUtils.getProps(housekeeperProperties); housekeeperReprocessingConfigs = PropertyUtils.getProps(housekeeperReprocessingProperties); edgeConfigs = PropertyUtils.getProps(edgeProperties); + edgeEventConfigs = PropertyUtils.getProps(edgeEventProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 6c758409d8..d563307390 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -285,7 +285,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng } @Override - public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId) { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 4f9c130986..056fbb2508 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -265,7 +265,7 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId) { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index b3c2a2f44c..44b2469da8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -207,7 +207,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId) { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 0c5cc94a72..205dbea3af 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -93,6 +93,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin housekeeperAdmin; private final TbQueueAdmin housekeeperReprocessingAdmin; private final TbQueueAdmin edgeAdmin; + private final TbQueueAdmin edgeEventAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -131,6 +132,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); + this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); } @Override @@ -466,14 +468,14 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi } @Override - public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId) { TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(topicService.buildTopicName("tb_edge_event.notifications." + tenantId + "." + edgeId + "." + serviceId)); + consumerBuilder.topic(topicService.buildTopicName("tb_edge_event.notifications." + tenantId + "." + edgeId)); consumerBuilder.clientId("monolith-to-edge-event-consumer" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId(topicService.buildTopicName("monolith-edge-event-consumer")); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeEventNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(edgeAdmin); + consumerBuilder.admin(edgeEventAdmin); consumerBuilder.statsService(consumerStatsService); return consumerBuilder.build(); } @@ -484,7 +486,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-to-edge-event-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(topicService.buildTopicName("edge-events")); - requestBuilder.admin(edgeAdmin); + requestBuilder.admin(edgeEventAdmin); return requestBuilder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 0866f1cde7..c462bf816e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -92,6 +92,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin housekeeperAdmin; private final TbQueueAdmin housekeeperReprocessingAdmin; private final TbQueueAdmin edgeAdmin; + private final TbQueueAdmin edgeEventAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -131,6 +132,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); + this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); } @Override @@ -415,14 +417,14 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId) { TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(topicService.buildTopicName("tb_edge_event.notifications." + tenantId + "." + edgeId + "." + serviceId)); + consumerBuilder.topic(topicService.buildTopicName("tb_edge_event.notifications." + tenantId + "." + edgeId)); consumerBuilder.clientId("tb-core-edge-event-consumer" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId(topicService.buildTopicName("tb-core-edge-event-consumer")); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeEventNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(edgeAdmin); + consumerBuilder.admin(edgeEventAdmin); consumerBuilder.statsService(consumerStatsService); return consumerBuilder.build(); } @@ -433,7 +435,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-edge-event-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(topicService.buildTopicName("edge-events")); - requestBuilder.admin(edgeAdmin); + requestBuilder.admin(edgeEventAdmin); return requestBuilder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 7090b87a66..7e9631979c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -80,6 +80,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueAdmin fwUpdatesAdmin; private final TbQueueAdmin housekeeperAdmin; private final TbQueueAdmin edgeAdmin; + private final TbQueueAdmin edgeEventAdmin; private final AtomicLong consumerCount = new AtomicLong(); public KafkaTbRuleEngineQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -109,6 +110,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); + this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); } @Override @@ -197,7 +199,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-edge-event-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(topicService.buildTopicName("edge-events")); - requestBuilder.admin(edgeAdmin); + requestBuilder.admin(edgeEventAdmin); return requestBuilder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index 7859e4f6fa..f0bb294e20 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -286,7 +286,7 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng } @Override - public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId) { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 3a0db39079..f885f1070e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -258,7 +258,7 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId) { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index d6ec066db8..a444276d19 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -284,7 +284,7 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId) { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index 0896774c33..053d682580 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -174,7 +174,7 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId) { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index c686d713f2..2772dd4ff9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -283,7 +283,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul } @Override - public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId) { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index 09abff3ece..640019a8a7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -258,7 +258,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId) { + public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId) { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 63fdce09e5..2fa95443ac 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -151,7 +151,7 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, Hous TbQueueProducer> createEdgeNotificationsMsgProducer(); - TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId, String serviceId); + TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId); TbQueueProducer> createEdgeEventMsgProducer(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java index 1e0202bf79..27fba18617 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java @@ -251,6 +251,7 @@ public class EdgeServiceImpl extends AbstractCachedEntityService Date: Wed, 30 Oct 2024 17:26:31 +0200 Subject: [PATCH 18/71] Process edge-event consumer with correct polling strategy --- .../service/edge/EdgeContextComponent.java | 3 +- .../edge/EdgeEventSourcingListener.java | 31 ++++-- .../edge/rpc/AbstractEdgeGrpcSession.java | 3 +- .../service/edge/rpc/EdgeGrpcService.java | 66 ++++++------ .../edge/rpc/KafkaEdgeEventService.java | 3 - .../edge/rpc/KafkaEdgeGrpcSession.java | 101 +++++++++--------- .../edge/rpc/PostgresEdgeGrpcSession.java | 1 - .../src/main/resources/thingsboard.yml | 2 +- 8 files changed, 107 insertions(+), 103 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java index 79259c36b6..ee50bbb4a3 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.edge; import lombok.Data; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.cache.limits.RateLimitService; @@ -141,7 +142,7 @@ public class EdgeContextComponent { @Autowired private EdgeRequestsService edgeRequestsService; - @Autowired + @Autowired(required = false) private EdgeRpcService edgeRpcService; @Autowired diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index 7706244680..ff5f62b655 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -53,8 +53,10 @@ import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; /** * This event listener does not support async event processing because relay on ThreadLocal @@ -77,11 +79,14 @@ import org.thingsboard.server.queue.discovery.TopicService; public class EdgeEventSourcingListener { private final TopicService topicService; - private final TbQueueAdmin tbQueueAdmin; - private final TenantService tenantService; private final TbClusterService tbClusterService; + + private final TenantService tenantService; private final EdgeSynchronizationManager edgeSynchronizationManager; + private final TbKafkaSettings kafkaSettings; + private final TbKafkaTopicConfigs kafkaTopicConfigs; + @Value("#{'${queue.type:null}' == 'kafka'}") private boolean isKafkaSupported; @@ -117,17 +122,13 @@ public class EdgeEventSourcingListener { return; } try { - if (EntityType.EDGE.equals(entityType)) { - if (isKafkaSupported) { - String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, (EdgeId) event.getEntityId()).getTopic(); - tbQueueAdmin.deleteTopic(topic); - } else { - return; - } - } if (EntityType.TENANT.equals(entityType)) { return; } + if (EntityType.EDGE.equals(entityType)) { + handleEdgeEntityDeletion((EdgeId) event.getEntityId(), tenantId); + return; + } log.trace("[{}] DeleteEntityEvent called: {}", tenantId, event); EdgeEventType type = getEdgeEventTypeForEntityEvent(event.getEntity()); EdgeEventActionType actionType = getEdgeEventActionTypeForEntityEvent(event.getEntity()); @@ -139,6 +140,14 @@ public class EdgeEventSourcingListener { } } + private void handleEdgeEntityDeletion(EdgeId edgeId, TenantId tenantId) { + if (isKafkaSupported) { + String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); + TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + kafkaAdmin.deleteTopic(topic); + } + } + private EdgeEventActionType getEdgeEventActionTypeForEntityEvent(Object entity) { if (entity instanceof AlarmComment) { return EdgeEventActionType.DELETED_COMMENT; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java index 397e1f6137..475826fee7 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java @@ -76,6 +76,7 @@ import org.thingsboard.server.gen.edge.v1.UserCredentialsRequestMsg; import org.thingsboard.server.gen.edge.v1.WidgetBundleTypesRequestMsg; import org.thingsboard.server.service.edge.EdgeContextComponent; import org.thingsboard.server.service.edge.rpc.fetch.EdgeEventFetcher; +import org.thingsboard.server.service.edge.rpc.fetch.GeneralEdgeEventFetcher; import org.thingsboard.server.service.edge.rpc.processor.alarm.AlarmProcessor; import org.thingsboard.server.service.edge.rpc.processor.asset.AssetProcessor; import org.thingsboard.server.service.edge.rpc.processor.asset.profile.AssetProfileProcessor; @@ -147,8 +148,6 @@ public abstract class AbstractEdgeGrpcSession processEdgeEvents() throws Exception; - public void initInputStream() { this.inputStream = new StreamObserver<>() { @Override diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java index bc63eb251a..bed4272ec1 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java @@ -89,7 +89,7 @@ import java.util.function.Consumer; @TbCoreComponent public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase implements EdgeRpcService { - private final ConcurrentMap sessions = new ConcurrentHashMap<>(); + private final ConcurrentMap> sessions = new ConcurrentHashMap<>(); private final ConcurrentMap sessionNewEventsLocks = new ConcurrentHashMap<>(); private final Map sessionNewEvents = new HashMap<>(); private final ConcurrentMap> sessionEdgeEventChecks = new ConcurrentHashMap<>(); @@ -212,22 +212,16 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i @Override public StreamObserver handleMsgs(StreamObserver outputStream) { - if (isKafkaSupported) { - return new KafkaEdgeGrpcSession(ctx, - outputStream, - this::onEdgeConnect, - this::onEdgeDisconnect, - sendDownlinkExecutorService, - this.maxInboundMessageSize, - this.maxHighPriorityQueueSizePerSession).getInputStream(); - } - return new PostgresEdgeGrpcSession(ctx, - outputStream, - this::onEdgeConnect, - this::onEdgeDisconnect, - sendDownlinkExecutorService, - this.maxInboundMessageSize, - this.maxHighPriorityQueueSizePerSession).getInputStream(); + AbstractEdgeGrpcSession session = createEdgeGrpcSession(outputStream); + return session.getInputStream(); + } + + private AbstractEdgeGrpcSession createEdgeGrpcSession(StreamObserver outputStream) { + return isKafkaSupported + ? new KafkaEdgeGrpcSession(ctx, outputStream, this::onEdgeConnect, this::onEdgeDisconnect, + sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession) + : new PostgresEdgeGrpcSession(ctx, outputStream, this::onEdgeConnect, this::onEdgeDisconnect, + sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession); } @Override @@ -258,7 +252,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i @Override public void updateEdge(TenantId tenantId, Edge edge) { - AbstractEdgeGrpcSession session = sessions.get(edge.getId()); + AbstractEdgeGrpcSession session = sessions.get(edge.getId()); if (session != null && session.isConnected()) { log.debug("[{}] Updating configuration for edge [{}] [{}]", tenantId, edge.getName(), edge.getId()); session.onConfigurationUpdate(edge); @@ -269,7 +263,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i @Override public void deleteEdge(TenantId tenantId, EdgeId edgeId) { - AbstractEdgeGrpcSession session = sessions.get(edgeId); + AbstractEdgeGrpcSession session = sessions.get(edgeId); if (session != null && session.isConnected()) { log.info("[{}] Closing and removing session for edge [{}]", tenantId, edgeId); session.close(); @@ -286,7 +280,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } private void onEdgeEventUpdate(TenantId tenantId, EdgeId edgeId) { - AbstractEdgeGrpcSession session = sessions.get(edgeId); + AbstractEdgeGrpcSession session = sessions.get(edgeId); if (session != null && session.isConnected()) { log.trace("[{}] onEdgeEventUpdate [{}]", tenantId, edgeId.getId()); updateSessionEventsFlag(tenantId, edgeId); @@ -297,7 +291,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i TenantId tenantId = msg.getTenantId(); EdgeEvent edgeEvent = msg.getEdgeEvent(); EdgeId edgeId = edgeEvent.getEdgeId(); - AbstractEdgeGrpcSession session = sessions.get(edgeId); + AbstractEdgeGrpcSession session = sessions.get(edgeId); if (session != null && session.isConnected()) { log.trace("[{}] onEdgeEvent [{}]", tenantId, edgeId); session.addEventToHighPriorityQueue(edgeEvent); @@ -318,7 +312,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } } - private void onEdgeConnect(EdgeId edgeId, AbstractEdgeGrpcSession edgeGrpcSession) { + private void onEdgeConnect(EdgeId edgeId, AbstractEdgeGrpcSession edgeGrpcSession) { Edge edge = edgeGrpcSession.getEdge(); TenantId tenantId = edge.getTenantId(); log.info("[{}][{}] edge [{}] connected successfully.", tenantId, edgeGrpcSession.getSessionId(), edgeId); @@ -334,17 +328,24 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i long lastConnectTs = System.currentTimeMillis(); save(tenantId, edgeId, DefaultDeviceStateService.LAST_CONNECT_TIME, lastConnectTs); edgeIdServiceIdCache.put(edgeId, serviceInfoProvider.getServiceId()); - if (isKafkaSupported) { - TbQueueConsumer> consumer = tbCoreQueueFactory.createEdgeEventMsgConsumer(tenantId, edgeId); - ((KafkaEdgeGrpcSession) edgeGrpcSession).initConsumer(consumer); + if (edgeGrpcSession instanceof KafkaEdgeGrpcSession) { + initializeKafkaConsumer((KafkaEdgeGrpcSession) edgeGrpcSession, tenantId, edgeId); } pushRuleEngineMessage(tenantId, edge, lastConnectTs, TbMsgType.CONNECT_EVENT); cancelScheduleEdgeEventsCheck(edgeId); - scheduleEdgeEventsCheck(edgeGrpcSession); + if (edgeGrpcSession instanceof PostgresEdgeGrpcSession) { + scheduleEdgeEventsCheck((PostgresEdgeGrpcSession) edgeGrpcSession); + } + } + + private void initializeKafkaConsumer(KafkaEdgeGrpcSession kafkaEdgeGrpcSession, TenantId tenantId, EdgeId edgeId) { + TbQueueConsumer> consumer = tbCoreQueueFactory.createEdgeEventMsgConsumer(tenantId, edgeId); + kafkaEdgeGrpcSession.initConsumer(() -> consumer, schedulerPoolSize); + kafkaEdgeGrpcSession.startConsumers(); } private void startSyncProcess(TenantId tenantId, EdgeId edgeId, UUID requestId, String requestServiceId) { - AbstractEdgeGrpcSession session = sessions.get(edgeId); + AbstractEdgeGrpcSession session = sessions.get(edgeId); if (session != null) { if (!session.isSyncCompleted()) { clusterService.pushEdgeSyncResponseToCore(new FromEdgeSyncResponse(requestId, tenantId, edgeId, false, "Sync process is active at the moment"), requestServiceId); @@ -364,7 +365,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i ToEdgeSyncRequest request = new ToEdgeSyncRequest(UUID.randomUUID(), tenantId, edgeId, serviceInfoProvider.getServiceId()); UUID requestId = request.getId(); - AbstractEdgeGrpcSession session = sessions.get(request.getEdgeId()); + AbstractEdgeGrpcSession session = sessions.get(request.getEdgeId()); if (session != null && !session.isSyncCompleted()) { responseConsumer.accept(new FromEdgeSyncResponse(requestId, request.getTenantId(), request.getEdgeId(), false, "Sync process is active at the moment")); } else { @@ -398,7 +399,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } } - private void scheduleEdgeEventsCheck(AbstractEdgeGrpcSession session) { + private void scheduleEdgeEventsCheck(PostgresEdgeGrpcSession session) { EdgeId edgeId = session.getEdge().getId(); UUID tenantId = session.getEdge().getTenantId().getId(); if (sessions.containsKey(edgeId)) { @@ -410,7 +411,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i if (Boolean.TRUE.equals(sessionNewEvents.get(edgeId))) { log.trace("[{}][{}] Set session new events flag to false", tenantId, edgeId.getId()); sessionNewEvents.put(edgeId, false); - Futures.addCallback(session.processEdgeEvents(), new FutureCallback() { + Futures.addCallback(session.processEdgeEvents(), new FutureCallback<>() { @Override public void onSuccess(Boolean newEventsAdded) { if (Boolean.TRUE.equals(newEventsAdded)) { @@ -418,6 +419,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } scheduleEdgeEventsCheck(session); } + @Override public void onFailure(Throwable t) { log.warn("[{}] Failed to process edge events for edge [{}]!", tenantId, session.getEdge().getId().getId(), t); @@ -456,7 +458,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i private void onEdgeDisconnect(Edge edge, UUID sessionId) { EdgeId edgeId = edge.getId(); log.info("[{}][{}] edge disconnected!", edgeId, sessionId); - AbstractEdgeGrpcSession toRemove = sessions.get(edgeId); + AbstractEdgeGrpcSession toRemove = sessions.get(edgeId); if (toRemove.getSessionId().equals(sessionId)) { toRemove = sessions.remove(edgeId); final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock()); @@ -506,6 +508,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } private static class AttributeSaveCallback implements FutureCallback { + private final TenantId tenantId; private final EdgeId edgeId; private final String key; @@ -527,6 +530,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i public void onFailure(Throwable t) { log.warn("[{}][{}] Failed to update attribute [{}] with value [{}]", tenantId, edgeId, key, value, t); } + } private void pushRuleEngineMessage(TenantId tenantId, Edge edge, long ts, TbMsgType msgType) { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java index 3caea5476a..13255dc4dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java @@ -35,7 +35,6 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeEventService; -import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -51,7 +50,6 @@ import java.util.UUID; public class KafkaEdgeEventService implements EdgeEventService { private final RateLimitService rateLimitService; - private final ApplicationEventPublisher eventPublisher; private final DataValidator edgeEventValidator; @Lazy private final TbQueueProducerProvider producerProvider; @@ -71,7 +69,6 @@ public class KafkaEdgeEventService implements EdgeEventService { ToEdgeEventNotificationMsg msg = ToEdgeEventNotificationMsg.newBuilder().setEdgeEventMsg(ProtoUtils.toProto(edgeEvent)).build(); producerProvider.getTbEdgeEventsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); - eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(edgeEvent.getTenantId()).entity(edgeEvent).entityId(edgeEvent.getEdgeId()).build()); return Futures.immediateFuture(null); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java index b7857d4e7d..e60523397f 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java @@ -17,33 +17,38 @@ package org.thingsboard.server.service.edge.rpc; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; import io.grpc.stub.StreamObserver; import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.Nullable; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.ResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; import org.thingsboard.server.service.edge.EdgeContextComponent; import java.util.ArrayList; import java.util.List; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.function.BiConsumer; +import java.util.function.Supplier; @Slf4j public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession { - private TbQueueConsumer> edgeEventsConsumer; + private ExecutorService consumerExecutor; + + private QueueConsumerManager> consumer; public KafkaEdgeGrpcSession(EdgeContextComponent ctx, StreamObserver outputStream, BiConsumer sessionOpenListener, @@ -53,75 +58,65 @@ public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession processEdgeEvents() { + protected void initConsumer(Supplier>> edgeEventsConsumer, long schedulerPoolSize) { + this.consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("edge-event-consumer")); + this.consumer = QueueConsumerManager.>builder() + .name("TB Edge events") + .msgPackProcessor(this::processMsgs) + .pollInterval(schedulerPoolSize) + .consumerCreator(edgeEventsConsumer) + .consumerExecutor(consumerExecutor) + .threadPrefix("edge-events") + .build(); + } + + private void processMsgs(List> msgs, TbQueueConsumer> consumer) { log.trace("[{}][{}] starting processing edge events", this.tenantId, this.sessionId); if (isConnected() && isSyncCompleted()) { if (!highPriorityQueue.isEmpty()) { processHighPriorityEvents(); } else { - return processKafkaEvents(); + List edgeEvents = new ArrayList<>(); + for (TbProtoQueueMsg msg : msgs) { + EdgeEvent edgeEvent = ProtoUtils.fromProto(msg.getValue().getEdgeEventMsg()); + edgeEvents.add(edgeEvent); + } + List downlinkMsgsPack = convertToDownlinkMsgsPack(edgeEvents); + Futures.addCallback(sendDownlinkMsgsPack(downlinkMsgsPack), new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Boolean isInterrupted) { + if (Boolean.TRUE.equals(isInterrupted)) { + log.debug("[{}][{}][{}] Send downlink messages task was interrupted", tenantId, edge.getId(), sessionId); + } else { + consumer.commit(); + } + } + @Override + public void onFailure(Throwable t) { + log.error("[{}] Failed to send downlink msgs pack", sessionId, t); + } + }, ctx.getGrpcCallbackExecutorService()); } } else { log.trace("[{}][{}] edge is not connected or sync is not completed. Skipping iteration", tenantId, sessionId); } - return null; } - private ListenableFuture processKafkaEvents() { - List edgeEvents = new ArrayList<>(); - try { - edgeEventsConsumer.subscribe(); - List> messages = edgeEventsConsumer.poll(100); - - if (messages.isEmpty()) { - return Futures.immediateFuture(Boolean.FALSE); - } - - for (TbProtoQueueMsg msg : messages) { - EdgeEvent edgeEvent = ProtoUtils.fromProto(msg.getValue().getEdgeEventMsg()); - edgeEvents.add(edgeEvent); - } - - List downlinkMsgsPack = convertToDownlinkMsgsPack(edgeEvents); - Futures.addCallback(sendDownlinkMsgsPack(downlinkMsgsPack), new FutureCallback<>() { - @Override - public void onSuccess(@Nullable Boolean isInterrupted) { - if (Boolean.TRUE.equals(isInterrupted)) { - log.debug("[{}][{}][{}] Send downlink messages task was interrupted", tenantId, edge.getId(), sessionId); - } else { - edgeEventsConsumer.commit(); - processEdgeEvents(); - } - } - @Override - public void onFailure(Throwable t) { - log.error("[{}] Failed to send downlink msgs pack", sessionId, t); - } - }, ctx.getGrpcCallbackExecutorService()); - return Futures.immediateFuture(Boolean.TRUE); - } catch (Exception e) { - log.error("[{}][{}] Error occurred while polling edge events from Kafka: {}", tenantId, edge.getId(), e.getMessage()); - return Futures.immediateFailedFuture(e); - } - } - - protected void initConsumer(TbQueueConsumer> edgeEventsConsumer) { - this.edgeEventsConsumer = edgeEventsConsumer; + public void startConsumers() { + consumer.subscribe(); + consumer.launch(); } @PreDestroy private void destroy() { - if (edgeEventsConsumer != null) { - edgeEventsConsumer.unsubscribe(); - } + consumer.stop(); + consumerExecutor.shutdown(); } public void stopConsumer() { - if (edgeEventsConsumer != null) { - edgeEventsConsumer.stop(); - } + consumer.stop(); + consumerExecutor.shutdown(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java index 8293942523..4d0fda21f6 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java @@ -63,7 +63,6 @@ public class PostgresEdgeGrpcSession extends AbstractEdgeGrpcSession processEdgeEvents() throws Exception { SettableFuture result = SettableFuture.create(); log.trace("[{}][{}] starting processing edge events", this.tenantId, this.sessionId); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 180a978744..037957d1c4 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1439,7 +1439,7 @@ swagger: # Queue configuration parameters queue: - type: "${TB_QUEUE_TYPE:kafka}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) + type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). in_memory: stats: From 50a55d15a9332c6cbdd8240eac92741e4ca6f071 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Thu, 31 Oct 2024 18:12:16 +0200 Subject: [PATCH 19/71] Add poll properties for edge-event kafka poll --- .../edge/EdgeEventSourcingListener.java | 10 ++++++++-- .../edge/rpc/KafkaEdgeEventService.java | 5 +++-- .../edge/rpc/KafkaEdgeGrpcSession.java | 2 +- .../ttl/KafkaEdgeTopicsCleanUpService.java | 12 +++++++++++- .../src/main/resources/thingsboard.yml | 7 ++++++- .../server/dao/edge/EdgeEventService.java | 1 + .../server/queue/kafka/TbKafkaAdmin.java | 19 +++++++++++++++++-- .../server/queue/kafka/TbKafkaSettings.java | 15 +++++++++++++++ .../provider/KafkaMonolithQueueFactory.java | 2 +- .../provider/KafkaTbCoreQueueFactory.java | 2 +- .../KafkaTbRuleEngineQueueFactory.java | 2 +- .../server/dao/edge/EdgeServiceImpl.java | 1 - ...ice.java => PostgresEdgeEventService.java} | 9 ++++++--- .../server/dao/model/sql/EdgeEventEntity.java | 1 + 14 files changed, 72 insertions(+), 16 deletions(-) rename dao/src/main/java/org/thingsboard/server/dao/edge/{BaseEdgeEventService.java => PostgresEdgeEventService.java} (93%) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index ff5f62b655..2d3f40b078 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -20,8 +20,10 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.common.util.JacksonUtil; @@ -84,8 +86,12 @@ public class EdgeEventSourcingListener { private final TenantService tenantService; private final EdgeSynchronizationManager edgeSynchronizationManager; - private final TbKafkaSettings kafkaSettings; - private final TbKafkaTopicConfigs kafkaTopicConfigs; + @Autowired(required = false) + @Lazy + private TbKafkaSettings kafkaSettings; + @Autowired(required = false) + @Lazy + private TbKafkaTopicConfigs kafkaTopicConfigs; @Value("#{'${queue.type:null}' == 'kafka'}") private boolean isKafkaSupported; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java index 13255dc4dc..77760524a5 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java @@ -20,7 +20,6 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.limits.RateLimitService; @@ -65,6 +64,7 @@ public class KafkaEdgeEventService implements EdgeEventService { throw new TbRateLimitsException(EntityType.EDGE); } edgeEventValidator.validate(edgeEvent, EdgeEvent::getTenantId); + TopicPartitionInfo tpi = topicService.getEdgeEventNotificationsTopic(edgeEvent.getTenantId(), edgeEvent.getEdgeId()); ToEdgeEventNotificationMsg msg = ToEdgeEventNotificationMsg.newBuilder().setEdgeEventMsg(ProtoUtils.toProto(edgeEvent)).build(); producerProvider.getTbEdgeEventsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); @@ -78,6 +78,7 @@ public class KafkaEdgeEventService implements EdgeEventService { } @Override - public void cleanupEvents(long ttl) {} + public void cleanupEvents(long ttl) { + } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java index e60523397f..b86aa0ecf2 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java @@ -89,7 +89,7 @@ public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession { PageDataIterable tenants = new PageDataIterable<>(tenantService::findTenantsIds, 10_000); @@ -101,6 +105,12 @@ public class KafkaEdgeTopicsCleanUpService { } } } +// String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edge.getId()).getTopic(); +// TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); +// if (kafkaAdmin.isTopicEmpty(topic)) { +// kafkaAdmin.deleteTopic(topic); +// log.info("Removed outdated topic for tenant {} and edge {} older than {}", tenantId, edge.getName(), Date.from(Instant.ofEpochMilli(currentTimeMillis - ONE_MONTH_MILLIS))); +// } } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 037957d1c4..448d5964f0 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1517,6 +1517,11 @@ queue: # tb_rule_engine.sq: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" + tb_edge_event.notifications: + # Example of specific consumer properties value per topic for edge event + - key: max.poll.records + # Example of specific consumer properties value per topic for edge event + value: "${TB_QUEUE_KAFKA_EDGE_EVENT_MAX_POLL_INTERVAL_MS:50}" tb_housekeeper: # Consumer properties for Housekeeper tasks topic - key: max.poll.records @@ -1556,7 +1561,7 @@ queue: edge: "${TB_QUEUE_KAFKA_EDGE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Edge event topic # TODO : discuss and adjust properties - edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:2592000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:60000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeEventService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeEventService.java index 2da05078ac..9a7fc854d1 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeEventService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeEventService.java @@ -33,4 +33,5 @@ public interface EdgeEventService { * @param ttl the ttl for edge events in seconds */ void cleanupEvents(long ttl); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index d614fe4fdd..86fa0dad3c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -17,7 +17,10 @@ package org.thingsboard.server.queue.kafka; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.ListOffsetsResult; import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.admin.OffsetSpec; +import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.TopicExistsException; @@ -174,8 +177,20 @@ public class TbKafkaAdmin implements TbQueueAdmin { public boolean isTopicEmpty(String topic) { try { - Map offsets = settings.getAdminClient().listConsumerGroupOffsets("__consumer_offsets").partitionsToOffsetAndMetadata().get(); - return offsets.entrySet().stream().noneMatch(entry -> entry.getKey().topic().equals(topic)); + TopicDescription topicDescription = settings.getAdminClient().describeTopics(Collections.singletonList(topic)).topicNameValues().get(topic).get(); + TopicPartition topicPartition = new TopicPartition(topic, topicDescription.partitions().get(0).partition()); + + // Get the earliest and latest offsets + Map beginningOffsets = + settings.getAdminClient().listOffsets(Collections.singletonMap(topicPartition, OffsetSpec.earliest())).all().get(); + Map endOffsets = + settings.getAdminClient().listOffsets(Collections.singletonMap(topicPartition, OffsetSpec.latest())).all().get(); + + long beginningOffset = beginningOffsets.get(topicPartition).offset(); + long endOffset = endOffsets.get(topicPartition).offset(); + + // If beginning and end offsets are both 0, the topic is empty + return beginningOffset == 0 && endOffset == 0; } catch (InterruptedException | ExecutionException e) { log.error("Failed to check if topic [{}] is empty.", topic, e); return false; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index 11b7df1958..5159b6f769 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -50,6 +50,8 @@ import java.util.Properties; @Component public class TbKafkaSettings { + private static final List BASIC_TOPIC_PREFIXES = List.of("tb_edge_event.notifications"); + @Value("${queue.kafka.bootstrap.servers}") private String servers; @@ -164,9 +166,22 @@ public class TbKafkaSettings { consumerPropertiesPerTopic .getOrDefault(topic, Collections.emptyList()) .forEach(kv -> props.put(kv.getKey(), kv.getValue())); + + applyBaseTopicProperties(props, topic); + return props; } + private void applyBaseTopicProperties(Properties props, String topic) { + if (topic != null) { + BASIC_TOPIC_PREFIXES.stream() + .filter(topic::startsWith) + .findFirst() + .ifPresent(prefix -> consumerPropertiesPerTopic.getOrDefault(prefix, Collections.emptyList()) + .forEach(kv -> props.put(kv.getKey(), kv.getValue()))); + } + } + public Properties toProducerProps() { Properties props = toProps(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, servers); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 205dbea3af..dd5d61e834 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -132,7 +132,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); - this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); + this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index c462bf816e..cc0e044917 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -132,7 +132,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); - this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); + this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 7e9631979c..74f95743b6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -110,7 +110,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); - this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); + this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java index 27fba18617..1e0202bf79 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java @@ -251,7 +251,6 @@ public class EdgeServiceImpl extends AbstractCachedEntityService() { @Override public void onSuccess(Void result) { - eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(edgeEvent.getTenantId()) - .entity(edgeEvent).entityId(edgeEvent.getEdgeId()).build()); + eventPublisher.publishEvent(SaveEntityEvent.builder() + .tenantId(edgeEvent.getTenantId()) + .entityId(edgeEvent.getEdgeId()) + .entity(edgeEvent) + .build()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EdgeEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EdgeEventEntity.java index 1489a40a27..9816938706 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EdgeEventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EdgeEventEntity.java @@ -129,4 +129,5 @@ public class EdgeEventEntity extends BaseSqlEntity implements BaseEnt private static long getTs(UUID uuid) { return (uuid.timestamp() - EPOCH_DIFF) / 10000; } + } From b981281205092194c65d7f39a09271a1d1123a38 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 1 Nov 2024 11:37:11 +0200 Subject: [PATCH 20/71] Fix tests --- .../service/edge/EdgeEventSourcingListener.java | 1 - .../service/ttl/KafkaEdgeTopicsCleanUpService.java | 12 +----------- application/src/main/resources/thingsboard.yml | 3 +-- .../thingsboard/server/queue/kafka/TbKafkaAdmin.java | 4 +--- 4 files changed, 3 insertions(+), 17 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index 2d3f40b078..3653eb3a40 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -77,7 +77,6 @@ import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; @Slf4j @Component @RequiredArgsConstructor -@ConditionalOnExpression("${edges.enabled:true}") public class EdgeEventSourcingListener { private final TopicService topicService; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java index 35a2578444..20e2ab6758 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java @@ -66,11 +66,7 @@ public class KafkaEdgeTopicsCleanUpService { private final ExecutorService executorService = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("kafka-edge-topic-cleanup")); - // @Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.edge_events.execution_interval_ms})}", fixedDelayString = "${sql.ttl.edge_events.execution_interval_ms}") - @Scheduled( - initialDelay = 60000, // 1 minute delay after startup - fixedDelayString = "${sql.ttl.edge_events.execution_interval_ms}" - ) + @Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.edge_events.execution_interval_ms})}", fixedDelayString = "${sql.ttl.edge_events.execution_interval_ms}") public void cleanUp() { executorService.submit(() -> { PageDataIterable tenants = new PageDataIterable<>(tenantService::findTenantsIds, 10_000); @@ -105,12 +101,6 @@ public class KafkaEdgeTopicsCleanUpService { } } } -// String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edge.getId()).getTopic(); -// TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); -// if (kafkaAdmin.isTopicEmpty(topic)) { -// kafkaAdmin.deleteTopic(topic); -// log.info("Removed outdated topic for tenant {} and edge {} older than {}", tenantId, edge.getName(), Date.from(Instant.ofEpochMilli(currentTimeMillis - ONE_MONTH_MILLIS))); -// } } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 448d5964f0..bf050d268a 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1560,8 +1560,7 @@ queue: # Kafka properties for Edge topic edge: "${TB_QUEUE_KAFKA_EDGE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Edge event topic - # TODO : discuss and adjust properties - edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:60000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:2592000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index 86fa0dad3c..c209c574e2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -180,7 +180,6 @@ public class TbKafkaAdmin implements TbQueueAdmin { TopicDescription topicDescription = settings.getAdminClient().describeTopics(Collections.singletonList(topic)).topicNameValues().get(topic).get(); TopicPartition topicPartition = new TopicPartition(topic, topicDescription.partitions().get(0).partition()); - // Get the earliest and latest offsets Map beginningOffsets = settings.getAdminClient().listOffsets(Collections.singletonMap(topicPartition, OffsetSpec.earliest())).all().get(); Map endOffsets = @@ -189,8 +188,7 @@ public class TbKafkaAdmin implements TbQueueAdmin { long beginningOffset = beginningOffsets.get(topicPartition).offset(); long endOffset = endOffsets.get(topicPartition).offset(); - // If beginning and end offsets are both 0, the topic is empty - return beginningOffset == 0 && endOffset == 0; + return beginningOffset == endOffset; } catch (InterruptedException | ExecutionException e) { log.error("Failed to check if topic [{}] is empty.", topic, e); return false; From e97668b01ce457e21c7ff9d4b925ff84b8582eb5 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 1 Nov 2024 17:53:11 +0200 Subject: [PATCH 21/71] Add logic to process postgres edge event fetcher on start --- .../edge/rpc/AbstractEdgeGrpcSession.java | 254 ++++++++++++++---- .../service/edge/rpc/EdgeGrpcService.java | 28 +- .../edge/rpc/PostgresEdgeGrpcSession.java | 155 ----------- .../ttl/KafkaEdgeTopicsCleanUpService.java | 12 +- .../server/dao/edge/EdgeService.java | 3 +- .../thingsboard/server/dao/edge/EdgeDao.java | 2 + .../server/dao/edge/EdgeServiceImpl.java | 8 + .../server/dao/sql/edge/EdgeRepository.java | 28 +- .../server/dao/sql/edge/JpaEdgeDao.java | 9 + 9 files changed, 262 insertions(+), 237 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java index 475826fee7..9aa21dd265 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java @@ -34,11 +34,14 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.limit.LimitedApi; import org.thingsboard.server.common.data.notification.rule.trigger.EdgeCommunicationFailureTrigger; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.SortOrder; +import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; @@ -89,6 +92,7 @@ import org.thingsboard.server.service.edge.rpc.processor.resource.ResourceProces import java.io.Closeable; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -103,27 +107,36 @@ import java.util.function.BiConsumer; @Data public abstract class AbstractEdgeGrpcSession> implements EdgeGrpcSession, Closeable { + private static final String QUEUE_START_TS_ATTR_KEY = "queueStartTs"; + private static final String QUEUE_START_SEQ_ID_ATTR_KEY = "queueStartSeqId"; + private static final int MAX_DOWNLINK_ATTEMPTS = 10; private static final String RATE_LIMIT_REACHED = "Rate limit reached"; protected static final ConcurrentLinkedQueue highPriorityQueue = new ConcurrentLinkedQueue<>(); - protected UUID sessionId; - protected BiConsumer sessionOpenListener; - protected BiConsumer sessionCloseListener; + private UUID sessionId; + private BiConsumer sessionOpenListener; + private BiConsumer sessionCloseListener; private final EdgeSessionState sessionState = new EdgeSessionState(); private final ReentrantLock downlinkMsgLock = new ReentrantLock(); - protected EdgeContextComponent ctx; - protected Edge edge; - protected TenantId tenantId; + private EdgeContextComponent ctx; + private Edge edge; + private TenantId tenantId; + + private Long newStartTs; + private Long previousStartTs; + private Long newStartSeqId; + private Long previousStartSeqId; + private Long seqIdEnd; - protected StreamObserver inputStream; - protected StreamObserver outputStream; + private StreamObserver inputStream; + private StreamObserver outputStream; - protected boolean connected; - protected volatile boolean syncCompleted; + private boolean connected; + private volatile boolean syncCompleted; private EdgeVersion edgeVersion; private int maxInboundMessageSize; @@ -231,6 +244,52 @@ public abstract class AbstractEdgeGrpcSession> future = startProcessingEdgeEvents(next); + Futures.addCallback(future, new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Pair result) { + doSync(cursor); + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}][{}] Exception during sync process", tenantId, edge.getId(), t); + } + }, ctx.getGrpcCallbackExecutorService()); + } else { + log.info("[{}][{}] sync process completed", this.tenantId, edge.getId()); + DownlinkMsg syncCompleteDownlinkMsg = DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .setSyncCompletedMsg(SyncCompletedMsg.newBuilder().build()) + .build(); + Futures.addCallback(sendDownlinkMsgsPack(Collections.singletonList(syncCompleteDownlinkMsg)), new FutureCallback<>() { + @Override + public void onSuccess(Boolean isInterrupted) { + markSyncCompletedSendEdgeEventUpdate(); + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}][{}] Exception during sending sync complete", tenantId, edge.getId(), t); + markSyncCompletedSendEdgeEventUpdate(); + } + }, ctx.getGrpcCallbackExecutorService()); + } + } + protected void processEdgeEvents(EdgeEventFetcher fetcher, PageLink pageLink, SettableFuture> result) { try { if (!highPriorityQueue.isEmpty()) { @@ -327,57 +386,11 @@ public abstract class AbstractEdgeGrpcSession> future = startProcessingEdgeEvents(next); - Futures.addCallback(future, new FutureCallback<>() { - @Override - public void onSuccess(@Nullable Pair result) { - doSync(cursor); - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}][{}] Exception during sync process", tenantId, edge.getId(), t); - } - }, ctx.getGrpcCallbackExecutorService()); - } else { - log.info("[{}][{}] sync process completed", this.tenantId, edge.getId()); - DownlinkMsg syncCompleteDownlinkMsg = DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .setSyncCompletedMsg(SyncCompletedMsg.newBuilder().build()) - .build(); - Futures.addCallback(sendDownlinkMsgsPack(Collections.singletonList(syncCompleteDownlinkMsg)), new FutureCallback<>() { - @Override - public void onSuccess(Boolean isInterrupted) { - markSyncCompletedSendEdgeEventUpdate(); - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}][{}] Exception during sending sync complete", tenantId, edge.getId(), t); - markSyncCompletedSendEdgeEventUpdate(); - } - }, ctx.getGrpcCallbackExecutorService()); - } - } - protected ListenableFuture sendDownlinkMsgsPack(List downlinkMsgsPack) { interruptPreviousSendDownlinkMsgsTask(); @@ -535,6 +548,68 @@ public abstract class AbstractEdgeGrpcSession processEdgeEvents() throws Exception { + SettableFuture result = SettableFuture.create(); + log.trace("[{}][{}] starting processing edge events", this.tenantId, this.sessionId); + if (isConnected() && isSyncCompleted()) { + Pair startTsAndSeqId = getQueueStartTsAndSeqId().get(); + this.previousStartTs = startTsAndSeqId.getFirst(); + this.previousStartSeqId = startTsAndSeqId.getSecond(); + GeneralEdgeEventFetcher fetcher = new GeneralEdgeEventFetcher( + this.previousStartTs, + this.previousStartSeqId, + this.seqIdEnd, + false, + Integer.toUnsignedLong(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount()), + ctx.getEdgeEventService()); + Futures.addCallback(startProcessingEdgeEvents(fetcher), new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Pair newStartTsAndSeqId) { + if (newStartTsAndSeqId != null) { + ListenableFuture> updateFuture = updateQueueStartTsAndSeqId(newStartTsAndSeqId); + Futures.addCallback(updateFuture, new FutureCallback<>() { + @Override + public void onSuccess(@Nullable List list) { + log.debug("[{}][{}] queue offset was updated [{}]", tenantId, sessionId, newStartTsAndSeqId); + if (fetcher.isSeqIdNewCycleStarted()) { + seqIdEnd = fetcher.getSeqIdEnd(); + boolean newEventsAvailable = isNewEdgeEventsAvailable(); + result.set(newEventsAvailable); + } else { + seqIdEnd = null; + boolean newEventsAvailable = isSeqIdStartedNewCycle(); + if (!newEventsAvailable) { + newEventsAvailable = isNewEdgeEventsAvailable(); + } + result.set(newEventsAvailable); + } + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}][{}] Failed to update queue offset [{}]", tenantId, sessionId, newStartTsAndSeqId, t); + result.setException(t); + } + }, ctx.getGrpcCallbackExecutorService()); + } else { + log.trace("[{}][{}] newStartTsAndSeqId is null. Skipping iteration without db update", tenantId, sessionId); + result.set(null); + } + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}][{}] Failed to process events", tenantId, sessionId, t); + result.setException(t); + } + }, ctx.getGrpcCallbackExecutorService()); + } else { + log.trace("[{}][{}] edge is not connected or sync is not completed. Skipping iteration", tenantId, sessionId); + result.set(null); + } + return result; + } + protected List convertToDownlinkMsgsPack(List edgeEvents) { List result = new ArrayList<>(); for (EdgeEvent edgeEvent : edgeEvents) { @@ -569,6 +644,73 @@ public abstract class AbstractEdgeGrpcSession> getQueueStartTsAndSeqId() { + ListenableFuture> future = + ctx.getAttributesService().find(edge.getTenantId(), edge.getId(), AttributeScope.SERVER_SCOPE, Arrays.asList(QUEUE_START_TS_ATTR_KEY, QUEUE_START_SEQ_ID_ATTR_KEY)); + return Futures.transform(future, attributeKvEntries -> { + long startTs = 0L; + long startSeqId = 0L; + for (AttributeKvEntry attributeKvEntry : attributeKvEntries) { + if (QUEUE_START_TS_ATTR_KEY.equals(attributeKvEntry.getKey())) { + startTs = attributeKvEntry.getLongValue().isPresent() ? attributeKvEntry.getLongValue().get() : 0L; + } + if (QUEUE_START_SEQ_ID_ATTR_KEY.equals(attributeKvEntry.getKey())) { + startSeqId = attributeKvEntry.getLongValue().isPresent() ? attributeKvEntry.getLongValue().get() : 0L; + } + } + if (startSeqId == 0L) { + startSeqId = findStartSeqIdFromOldestEventIfAny(); + } + return Pair.of(startTs, startSeqId); + }, ctx.getGrpcCallbackExecutorService()); + } + + private boolean isSeqIdStartedNewCycle() { + try { + TimePageLink pageLink = new TimePageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount(), 0, null, null, this.newStartTs, System.currentTimeMillis()); + PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), 0L, this.previousStartSeqId == 0 ? null : this.previousStartSeqId - 1, pageLink); + return !edgeEvents.getData().isEmpty(); + } catch (Exception e) { + log.error("[{}][{}][{}] Failed to execute isSeqIdStartedNewCycle", this.tenantId, edge.getId(), sessionId, e); + } + return false; + } + + private boolean isNewEdgeEventsAvailable() { + try { + TimePageLink pageLink = new TimePageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount(), 0, null, null, this.newStartTs, System.currentTimeMillis()); + PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), this.newStartSeqId, null, pageLink); + return !edgeEvents.getData().isEmpty() || !highPriorityQueue.isEmpty(); + } catch (Exception e) { + log.error("[{}][{}][{}] Failed to execute isNewEdgeEventsAvailable", this.tenantId, edge.getId(), sessionId, e); + } + return false; + } + + private long findStartSeqIdFromOldestEventIfAny() { + long startSeqId = 0L; + try { + TimePageLink pageLink = new TimePageLink(1, 0, null, new SortOrder("createdTime"), null, null); + PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), null, null, pageLink); + if (!edgeEvents.getData().isEmpty()) { + startSeqId = edgeEvents.getData().get(0).getSeqId() - 1; + } + } catch (Exception e) { + log.error("[{}][{}][{}] Failed to execute findStartSeqIdFromOldestEventIfAny", this.tenantId, edge.getId(), sessionId, e); + } + return startSeqId; + } + + private ListenableFuture> updateQueueStartTsAndSeqId(Pair pair) { + this.newStartTs = pair.getFirst(); + this.newStartSeqId = pair.getSecond(); + log.trace("[{}] updateQueueStartTsAndSeqId [{}][{}][{}]", this.sessionId, edge.getId(), this.newStartTs, this.newStartSeqId); + List attributes = Arrays.asList( + new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_TS_ATTR_KEY, this.newStartTs), System.currentTimeMillis()), + new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_SEQ_ID_ATTR_KEY, this.newStartSeqId), System.currentTimeMillis())); + return ctx.getAttributesService().save(edge.getTenantId(), edge.getId(), AttributeScope.SERVER_SCOPE, attributes); + } + protected ListenableFuture> startProcessingEdgeEvents(EdgeEventFetcher fetcher) { SettableFuture> result = SettableFuture.create(); PageLink pageLink = fetcher.getPageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount()); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java index bed4272ec1..ddc70ea3c4 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java @@ -94,6 +94,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i private final Map sessionNewEvents = new HashMap<>(); private final ConcurrentMap> sessionEdgeEventChecks = new ConcurrentHashMap<>(); private final ConcurrentMap> localSyncEdgeRequests = new ConcurrentHashMap<>(); + private final ConcurrentMap edgeEventsProcessed = new ConcurrentHashMap<>(); @Value("${edges.rpc.port}") private int rpcPort; @@ -328,13 +329,18 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i long lastConnectTs = System.currentTimeMillis(); save(tenantId, edgeId, DefaultDeviceStateService.LAST_CONNECT_TIME, lastConnectTs); edgeIdServiceIdCache.put(edgeId, serviceInfoProvider.getServiceId()); - if (edgeGrpcSession instanceof KafkaEdgeGrpcSession) { - initializeKafkaConsumer((KafkaEdgeGrpcSession) edgeGrpcSession, tenantId, edgeId); - } pushRuleEngineMessage(tenantId, edge, lastConnectTs, TbMsgType.CONNECT_EVENT); cancelScheduleEdgeEventsCheck(edgeId); + edgeEventsProcessed.putIfAbsent(edgeId, Boolean.FALSE); + if (edgeGrpcSession instanceof KafkaEdgeGrpcSession session) { + Boolean isChecked = edgeEventsProcessed.get(edgeId); + if (Boolean.FALSE.equals(isChecked)) { + scheduleEdgeEventsCheck(session); + } + initializeKafkaConsumer(session, tenantId, edgeId); + } if (edgeGrpcSession instanceof PostgresEdgeGrpcSession) { - scheduleEdgeEventsCheck((PostgresEdgeGrpcSession) edgeGrpcSession); + scheduleEdgeEventsCheck(edgeGrpcSession); } } @@ -399,7 +405,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } } - private void scheduleEdgeEventsCheck(PostgresEdgeGrpcSession session) { + private void scheduleEdgeEventsCheck(AbstractEdgeGrpcSession session) { EdgeId edgeId = session.getEdge().getId(); UUID tenantId = session.getEdge().getTenantId().getId(); if (sessions.containsKey(edgeId)) { @@ -408,7 +414,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock()); newEventLock.lock(); try { - if (Boolean.TRUE.equals(sessionNewEvents.get(edgeId))) { + if (Boolean.TRUE.equals(sessionNewEvents.get(edgeId)) && Boolean.FALSE.equals(edgeEventsProcessed.get(edgeId))) { log.trace("[{}][{}] Set session new events flag to false", tenantId, edgeId.getId()); sessionNewEvents.put(edgeId, false); Futures.addCallback(session.processEdgeEvents(), new FutureCallback<>() { @@ -417,7 +423,11 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i if (Boolean.TRUE.equals(newEventsAdded)) { sessionNewEvents.put(edgeId, true); } - scheduleEdgeEventsCheck(session); + if (isKafkaSupported) { + edgeEventsProcessed.put(edgeId, true); + } else { + scheduleEdgeEventsCheck(session); + } } @Override @@ -427,7 +437,9 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } }, ctx.getGrpcCallbackExecutorService()); } else { - scheduleEdgeEventsCheck(session); + if (!isKafkaSupported) { + scheduleEdgeEventsCheck(session); + } } } finally { newEventLock.unlock(); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java index 4d0fda21f6..4ef55e5961 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java @@ -15,30 +15,13 @@ */ package org.thingsboard.server.service.edge.rpc; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.SettableFuture; import io.grpc.stub.StreamObserver; import lombok.extern.slf4j.Slf4j; -import org.checkerframework.checker.nullness.qual.Nullable; -import org.springframework.data.util.Pair; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.EdgeId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.SortOrder; -import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.gen.edge.v1.ResponseMsg; import org.thingsboard.server.service.edge.EdgeContextComponent; -import org.thingsboard.server.service.edge.rpc.fetch.GeneralEdgeEventFetcher; -import java.util.Arrays; -import java.util.List; import java.util.UUID; import java.util.concurrent.ScheduledExecutorService; import java.util.function.BiConsumer; @@ -46,15 +29,6 @@ import java.util.function.BiConsumer; @Slf4j public class PostgresEdgeGrpcSession extends AbstractEdgeGrpcSession { - private static final String QUEUE_START_TS_ATTR_KEY = "queueStartTs"; - private static final String QUEUE_START_SEQ_ID_ATTR_KEY = "queueStartSeqId"; - - private Long newStartTs; - private Long previousStartTs; - private Long newStartSeqId; - private Long previousStartSeqId; - private Long seqIdEnd; - PostgresEdgeGrpcSession(EdgeContextComponent ctx, StreamObserver outputStream, BiConsumer sessionOpenListener, BiConsumer sessionCloseListener, ScheduledExecutorService sendDownlinkExecutorService, @@ -63,133 +37,4 @@ public class PostgresEdgeGrpcSession extends AbstractEdgeGrpcSession processEdgeEvents() throws Exception { - SettableFuture result = SettableFuture.create(); - log.trace("[{}][{}] starting processing edge events", this.tenantId, this.sessionId); - if (isConnected() && isSyncCompleted()) { - Pair startTsAndSeqId = getQueueStartTsAndSeqId().get(); - this.previousStartTs = startTsAndSeqId.getFirst(); - this.previousStartSeqId = startTsAndSeqId.getSecond(); - GeneralEdgeEventFetcher fetcher = new GeneralEdgeEventFetcher( - this.previousStartTs, - this.previousStartSeqId, - this.seqIdEnd, - false, - Integer.toUnsignedLong(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount()), - ctx.getEdgeEventService()); - Futures.addCallback(startProcessingEdgeEvents(fetcher), new FutureCallback<>() { - @Override - public void onSuccess(@Nullable Pair newStartTsAndSeqId) { - if (newStartTsAndSeqId != null) { - ListenableFuture> updateFuture = updateQueueStartTsAndSeqId(newStartTsAndSeqId); - Futures.addCallback(updateFuture, new FutureCallback<>() { - @Override - public void onSuccess(@Nullable List list) { - log.debug("[{}][{}] queue offset was updated [{}]", tenantId, sessionId, newStartTsAndSeqId); - if (fetcher.isSeqIdNewCycleStarted()) { - seqIdEnd = fetcher.getSeqIdEnd(); - boolean newEventsAvailable = isNewEdgeEventsAvailable(); - result.set(newEventsAvailable); - } else { - seqIdEnd = null; - boolean newEventsAvailable = isSeqIdStartedNewCycle(); - if (!newEventsAvailable) { - newEventsAvailable = isNewEdgeEventsAvailable(); - } - result.set(newEventsAvailable); - } - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}][{}] Failed to update queue offset [{}]", tenantId, sessionId, newStartTsAndSeqId, t); - result.setException(t); - } - }, ctx.getGrpcCallbackExecutorService()); - } else { - log.trace("[{}][{}] newStartTsAndSeqId is null. Skipping iteration without db update", tenantId, sessionId); - result.set(null); - } - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}][{}] Failed to process events", tenantId, sessionId, t); - result.setException(t); - } - }, ctx.getGrpcCallbackExecutorService()); - } else { - log.trace("[{}][{}] edge is not connected or sync is not completed. Skipping iteration", tenantId, sessionId); - result.set(null); - } - return result; - } - - private ListenableFuture> getQueueStartTsAndSeqId() { - ListenableFuture> future = - ctx.getAttributesService().find(edge.getTenantId(), edge.getId(), AttributeScope.SERVER_SCOPE, Arrays.asList(QUEUE_START_TS_ATTR_KEY, QUEUE_START_SEQ_ID_ATTR_KEY)); - return Futures.transform(future, attributeKvEntries -> { - long startTs = 0L; - long startSeqId = 0L; - for (AttributeKvEntry attributeKvEntry : attributeKvEntries) { - if (QUEUE_START_TS_ATTR_KEY.equals(attributeKvEntry.getKey())) { - startTs = attributeKvEntry.getLongValue().isPresent() ? attributeKvEntry.getLongValue().get() : 0L; - } - if (QUEUE_START_SEQ_ID_ATTR_KEY.equals(attributeKvEntry.getKey())) { - startSeqId = attributeKvEntry.getLongValue().isPresent() ? attributeKvEntry.getLongValue().get() : 0L; - } - } - if (startSeqId == 0L) { - startSeqId = findStartSeqIdFromOldestEventIfAny(); - } - return Pair.of(startTs, startSeqId); - }, ctx.getGrpcCallbackExecutorService()); - } - - private boolean isSeqIdStartedNewCycle() { - try { - TimePageLink pageLink = new TimePageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount(), 0, null, null, this.newStartTs, System.currentTimeMillis()); - PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), 0L, this.previousStartSeqId == 0 ? null : this.previousStartSeqId - 1, pageLink); - return !edgeEvents.getData().isEmpty(); - } catch (Exception e) { - log.error("[{}][{}][{}] Failed to execute isSeqIdStartedNewCycle", this.tenantId, edge.getId(), sessionId, e); - } - return false; - } - - private boolean isNewEdgeEventsAvailable() { - try { - TimePageLink pageLink = new TimePageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount(), 0, null, null, this.newStartTs, System.currentTimeMillis()); - PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), this.newStartSeqId, null, pageLink); - return !edgeEvents.getData().isEmpty() || !highPriorityQueue.isEmpty(); - } catch (Exception e) { - log.error("[{}][{}][{}] Failed to execute isNewEdgeEventsAvailable", this.tenantId, edge.getId(), sessionId, e); - } - return false; - } - - private long findStartSeqIdFromOldestEventIfAny() { - long startSeqId = 0L; - try { - TimePageLink pageLink = new TimePageLink(1, 0, null, new SortOrder("createdTime"), null, null); - PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), null, null, pageLink); - if (!edgeEvents.getData().isEmpty()) { - startSeqId = edgeEvents.getData().get(0).getSeqId() - 1; - } - } catch (Exception e) { - log.error("[{}][{}][{}] Failed to execute findStartSeqIdFromOldestEventIfAny", this.tenantId, edge.getId(), sessionId, e); - } - return startSeqId; - } - - private ListenableFuture> updateQueueStartTsAndSeqId(Pair pair) { - this.newStartTs = pair.getFirst(); - this.newStartSeqId = pair.getSecond(); - log.trace("[{}] updateQueueStartTsAndSeqId [{}][{}][{}]", this.sessionId, edge.getId(), this.newStartTs, this.newStartSeqId); - List attributes = Arrays.asList( - new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_TS_ATTR_KEY, this.newStartTs), System.currentTimeMillis()), - new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_SEQ_ID_ATTR_KEY, this.newStartSeqId), System.currentTimeMillis())); - return ctx.getAttributesService().save(edge.getTenantId(), edge.getId(), AttributeScope.SERVER_SCOPE, attributes); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java index 20e2ab6758..eb56878d5e 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java @@ -22,7 +22,7 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.page.PageDataIterable; @@ -85,19 +85,19 @@ public class KafkaEdgeTopicsCleanUpService { return; } - PageDataIterable edges = new PageDataIterable<>(link -> edgeService.findEdgesByTenantId(tenantId, link), 1024); + PageDataIterable edgeIds = new PageDataIterable<>(link -> edgeService.findEdgeIdsByTenantId(tenantId, link), 1024); long currentTimeMillis = System.currentTimeMillis(); - for (Edge edge : edges) { - Optional attributeOpt = attributesService.find(tenantId, edge.getId(), AttributeScope.SERVER_SCOPE, DefaultDeviceStateService.LAST_CONNECT_TIME).get(); + for (EdgeId edgeId : edgeIds) { + Optional attributeOpt = attributesService.find(tenantId, edgeId, AttributeScope.SERVER_SCOPE, DefaultDeviceStateService.LAST_CONNECT_TIME).get(); if (attributeOpt.isPresent()) { Optional lastConnectTimeOpt = attributeOpt.get().getLongValue(); if (lastConnectTimeOpt.isPresent() && isTopicExpired(lastConnectTimeOpt.get(), currentTimeMillis)) { - String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edge.getId()).getTopic(); + String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); if (kafkaAdmin.isTopicEmpty(topic)) { kafkaAdmin.deleteTopic(topic); - log.info("Removed outdated topic for tenant {} and edge {} older than {}", tenantId, edge.getName(), Date.from(Instant.ofEpochMilli(currentTimeMillis - ONE_MONTH_MILLIS))); + log.info("Removed outdated topic for tenant {} and edge with id {} older than {}", tenantId, edgeId, Date.from(Instant.ofEpochMilli(currentTimeMillis - ONE_MONTH_MILLIS))); } } } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeService.java index c18f42ace7..d49a16e11f 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeService.java @@ -32,7 +32,6 @@ import org.thingsboard.server.dao.entity.EntityDaoService; import java.util.List; import java.util.Optional; -import java.util.UUID; public interface EdgeService extends EntityDaoService { @@ -56,6 +55,8 @@ public interface EdgeService extends EntityDaoService { void deleteEdge(TenantId tenantId, EdgeId edgeId); + PageData findEdgeIdsByTenantId(TenantId tenantId, PageLink pageLink); + PageData findEdgesByTenantId(TenantId tenantId, PageLink pageLink); PageData findEdgesByTenantIdAndType(TenantId tenantId, String type, PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java index be876cb5e0..e012766a8f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java @@ -40,6 +40,8 @@ public interface EdgeDao extends Dao { EdgeInfo findEdgeInfoById(TenantId tenantId, UUID edgeId); + PageData findEdgeIdsByTenantId(UUID tenantId, PageLink pageLink); + PageData findEdgesByTenantId(UUID tenantId, PageLink pageLink); PageData findEdgesByTenantIdAndType(UUID tenantId, String type, PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java index 1e0202bf79..c9b465b178 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java @@ -261,6 +261,14 @@ public class EdgeServiceImpl extends AbstractCachedEntityService findEdgeIdsByTenantId(TenantId tenantId, PageLink pageLink) { + log.trace("Executing findEdgeIdsByTenantId, tenantId [{}], pageLink [{}]", tenantId, pageLink); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validatePageLink(pageLink); + return edgeDao.findEdgeIdsByTenantId(tenantId.getId(), pageLink); + } + @Override public PageData findEdgesByTenantId(TenantId tenantId, PageLink pageLink) { log.trace("Executing findEdgesByTenantId, tenantId [{}], pageLink [{}]", tenantId, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java index b0c6a47783..79c3e93170 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java @@ -42,6 +42,12 @@ public interface EdgeRepository extends JpaRepository { "WHERE d.id = :edgeId") EdgeInfoEntity findEdgeInfoById(@Param("edgeId") UUID edgeId); + @Query("SELECT d.id FROM EdgeEntity d WHERE d.tenantId = :tenantId " + + "AND (:textSearch IS NULL OR ilike(d.name, CONCAT('%', :textSearch, '%')) = true)") + Page findIdsByTenantId(@Param("tenantId") UUID tenantId, + @Param("textSearch") String textSearch, + Pageable pageable); + @Query("SELECT d FROM EdgeEntity d WHERE d.tenantId = :tenantId " + "AND (:textSearch IS NULL OR ilike(d.name, CONCAT('%', :textSearch, '%')) = true)") Page findByTenantId(@Param("tenantId") UUID tenantId, @@ -93,9 +99,9 @@ public interface EdgeRepository extends JpaRepository { "AND a.customerId = :customerId " + "AND (:textSearch IS NULL OR ilike(a.name, CONCAT('%', :textSearch, '%')) = true)") Page findEdgeInfosByTenantIdAndCustomerId(@Param("tenantId") UUID tenantId, - @Param("customerId") UUID customerId, - @Param("textSearch") String textSearch, - Pageable pageable); + @Param("customerId") UUID customerId, + @Param("textSearch") String textSearch, + Pageable pageable); @Query("SELECT new org.thingsboard.server.dao.model.sql.EdgeInfoEntity(a, c.title, c.additionalInfo) " + "FROM EdgeEntity a " + @@ -105,10 +111,10 @@ public interface EdgeRepository extends JpaRepository { "AND a.type = :type " + "AND (:textSearch IS NULL OR ilike(a.name, CONCAT('%', :textSearch, '%')) = true)") Page findEdgeInfosByTenantIdAndCustomerIdAndType(@Param("tenantId") UUID tenantId, - @Param("customerId") UUID customerId, - @Param("type") String type, - @Param("textSearch") String textSearch, - Pageable pageable); + @Param("customerId") UUID customerId, + @Param("type") String type, + @Param("textSearch") String textSearch, + Pageable pageable); @Query("SELECT ee FROM EdgeEntity ee, RelationEntity re WHERE ee.tenantId = :tenantId " + "AND ee.id = re.fromId AND re.fromType = 'EDGE' AND re.relationTypeGroup = 'EDGE' " + @@ -125,10 +131,10 @@ public interface EdgeRepository extends JpaRepository { "AND re.relationType = 'Contains' AND re.toId = :entityId AND re.toType = :entityType " + "AND (:textSearch IS NULL OR ilike(ee.name, CONCAT('%', :textSearch, '%')) = true)") Page findIdsByTenantIdAndEntityId(@Param("tenantId") UUID tenantId, - @Param("entityId") UUID entityId, - @Param("entityType") String entityType, - @Param("textSearch") String textSearch, - Pageable pageable); + @Param("entityId") UUID entityId, + @Param("entityType") String entityType, + @Param("textSearch") String textSearch, + Pageable pageable); @Query("SELECT ee FROM EdgeEntity ee, TenantEntity te WHERE ee.tenantId = te.id AND te.tenantProfileId = :tenantProfileId ") Page findByTenantProfileId(@Param("tenantProfileId") UUID tenantProfileId, diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java index 3c2d64825f..b02a28fee4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java @@ -64,6 +64,15 @@ public class JpaEdgeDao extends JpaAbstractDao implements Edge return DaoUtil.getData(edgeRepository.findEdgeInfoById(edgeId)); } + @Override + public PageData findEdgeIdsByTenantId(UUID tenantId, PageLink pageLink) { + return DaoUtil.pageToPageData( + edgeRepository.findIdsByTenantId( + tenantId, + pageLink.getTextSearch(), + DaoUtil.toPageable(pageLink))).mapData(EdgeId::fromUUID); + } + @Override public PageData findEdgesByTenantId(UUID tenantId, PageLink pageLink) { return DaoUtil.toPageData( From 7b1eefbfb4b9fd81b3f2ae1a871951f618ae89ed Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Mon, 4 Nov 2024 12:53:54 +0200 Subject: [PATCH 22/71] Added ability to close transport session on RPC delivery timeout --- .../server/actors/ActorSystemContext.java | 4 +++ .../device/DeviceActorMessageProcessor.java | 32 ++++++++++++++----- .../src/main/resources/thingsboard.yml | 11 +++++++ common/proto/src/main/proto/queue.proto | 1 + .../transport/mqtt/MqttTransportHandler.java | 18 ++++------- 5 files changed, 46 insertions(+), 20 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 5966716ed8..25671940dd 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -565,6 +565,10 @@ public class ActorSystemContext { @Getter private String rpcSubmitStrategy; + @Value("${actors.rpc.close_session_on_rpc_delivery_timeout:false}") + @Getter + private boolean closeTransportSessionOnRpcDeliveryTimeout; + @Value("${actors.rpc.response_timeout_ms:30000}") @Getter private long rpcResponseTimeout; diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 3e8cd9592f..3a000bfe4d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -132,6 +132,7 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso private final boolean rpcSequential; private final RpcSubmitStrategy rpcSubmitStrategy; private final ScheduledExecutorService scheduler; + private final boolean closeTransportSessionOnRpcDeliveryTimeout; private int rpcSeq = 0; private String deviceName; @@ -145,6 +146,7 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso this.tenantId = tenantId; this.deviceId = deviceId; this.rpcSubmitStrategy = RpcSubmitStrategy.parse(systemContext.getRpcSubmitStrategy()); + this.closeTransportSessionOnRpcDeliveryTimeout = systemContext.isCloseTransportSessionOnRpcDeliveryTimeout(); this.rpcSequential = !rpcSubmitStrategy.equals(RpcSubmitStrategy.BURST); this.attributeSubscriptions = new HashMap<>(); this.rpcSubscriptions = new HashMap<>(); @@ -223,7 +225,7 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso log.error("[{}][{}][{}] Failed to save RPC request to edge queue {}", tenantId, deviceId, edgeId.getId(), request, e); } } else if (isSendNewRpcAvailable()) { - sent = rpcSubscriptions.size() > 0; + sent = !rpcSubscriptions.isEmpty(); Set syncSessionSet = new HashSet<>(); rpcSubscriptions.forEach((sessionId, sessionInfo) -> { log.debug("[{}][{}][{}][{}] send RPC request to transport ...", deviceId, sessionId, rpcId, requestId); @@ -598,7 +600,7 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso } void processAttributesUpdate(DeviceAttributesEventNotificationMsg msg) { - if (attributeSubscriptions.size() > 0) { + if (!attributeSubscriptions.isEmpty()) { boolean hasNotificationData = false; AttributeUpdateNotificationMsg.Builder notification = AttributeUpdateNotificationMsg.newBuilder(); if (msg.isDeleted()) { @@ -613,7 +615,7 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso } else { if (DataConstants.SHARED_SCOPE.equals(msg.getScope())) { List attributes = new ArrayList<>(msg.getValues()); - if (attributes.size() > 0) { + if (!attributes.isEmpty()) { List sharedUpdated = msg.getValues().stream().map(t -> KvProtoUtil.toTsKvProto(t.getLastUpdateTs(), t)) .collect(Collectors.toList()); if (!sharedUpdated.isEmpty()) { @@ -705,10 +707,19 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso maxRpcRetries = maxRpcRetries == null ? systemContext.getMaxRpcRetries() : Math.min(maxRpcRetries, systemContext.getMaxRpcRetries()); if (maxRpcRetries <= md.getRetries()) { - toDeviceRpcPendingMap.remove(requestId); - status = RpcStatus.FAILED; - response = JacksonUtil.newObjectNode().put("error", "There was a Timeout and all retry " + - "attempts have been exhausted. Retry attempts set: " + maxRpcRetries); + if (closeTransportSessionOnRpcDeliveryTimeout) { + md.setRetries(0); + status = RpcStatus.QUEUED; + sessions.forEach(this::notifyTransportAboutClosedSessionRpcDeliveryTimeout); + attributeSubscriptions.clear(); + rpcSubscriptions.clear(); + dumpSessions(); + } else { + toDeviceRpcPendingMap.remove(requestId); + status = RpcStatus.FAILED; + response = JacksonUtil.newObjectNode().put("error", "There was a Timeout and all retry " + + "attempts have been exhausted. Retry attempts set: " + maxRpcRetries); + } } else { md.setRetries(md.getRetries() + 1); } @@ -855,8 +866,13 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso } } + private void notifyTransportAboutClosedSessionRpcDeliveryTimeout(UUID sessionId, SessionInfoMetaData sessionMd) { + log.debug("Close session due to RPC delivery failure. sessionId: [{}] sessionMd: [{}]", sessionId, sessionMd); + notifyTransportAboutClosedSession(sessionId, sessionMd, "RPC delivery failed!", SessionCloseReason.RPC_DELIVERY_TIMEOUT); + } + private void notifyTransportAboutClosedSessionMaxSessionsLimit(UUID sessionId, SessionInfoMetaData sessionMd) { - log.debug("remove eldest session (max concurrent sessions limit reached per device) sessionId: [{}] sessionMd: [{}]", sessionId, sessionMd); + log.debug("Remove eldest session (max concurrent sessions limit reached per device) sessionId: [{}] sessionMd: [{}]", sessionId, sessionMd); notifyTransportAboutClosedSession(sessionId, sessionMd, "max concurrent sessions limit reached per device!", SessionCloseReason.MAX_CONCURRENT_SESSIONS_LIMIT_REACHED); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 106810a7a1..b4c178a864 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -484,6 +484,17 @@ actors: submit_strategy: "${ACTORS_RPC_SUBMIT_STRATEGY_TYPE:BURST}" # Time in milliseconds for RPC to receive a response after delivery. Used only for SEQUENTIAL_ON_RESPONSE_FROM_DEVICE submit strategy. response_timeout_ms: "${ACTORS_RPC_RESPONSE_TIMEOUT_MS:30000}" + # Close transport session if RPC delivery timed out. If enabled, RPC will be reverted to the queued state. + # Note: + # - For MQTT transport: + # - QoS level 0: This feature does not apply, as no acknowledgment is expected, and therefore no timeout is triggered. + # - QoS level 1: This feature applies, as an acknowledgment is expected. + # - QoS level 2: Unsupported. + # - For CoAP transport: + # - Confirmable requests: This feature applies, as delivery confirmation is expected. + # - Non-confirmable requests: This feature does not apply, as no delivery acknowledgment is expected. + # - For HTTP and SNPM transports: RPC is considered delivered immediately, and there is no logic to await acknowledgment. + close_session_on_rpc_delivery_timeout: "${ACTORS_RPC_CLOSE_SESSION_ON_RPC_DELIVERY_TIMEOUT:false}" statistics: # Enable/disable actor statistics enabled: "${ACTORS_STATISTICS_ENABLED:true}" diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index f84c7f7c9b..c85b9b9251 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -607,6 +607,7 @@ enum SessionCloseReason { CREDENTIALS_UPDATED = 1; MAX_CONCURRENT_SESSIONS_LIMIT_REACHED = 2; SESSION_TIMEOUT = 3; + RPC_DELIVERY_TIMEOUT = 4; } message SessionCloseNotificationProto { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index d7f64e58ee..a3e6d3fd73 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -1304,18 +1304,12 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement public void onRemoteSessionCloseCommand(UUID sessionId, TransportProtos.SessionCloseNotificationProto sessionCloseNotification) { log.trace("[{}] Received the remote command to close the session: {}", sessionId, sessionCloseNotification.getMessage()); transportService.deregisterSession(deviceSessionCtx.getSessionInfo()); - MqttReasonCodes.Disconnect returnCode = MqttReasonCodes.Disconnect.IMPLEMENTATION_SPECIFIC_ERROR; - switch (sessionCloseNotification.getReason()) { - case CREDENTIALS_UPDATED: - returnCode = MqttReasonCodes.Disconnect.ADMINISTRATIVE_ACTION; - break; - case MAX_CONCURRENT_SESSIONS_LIMIT_REACHED: - returnCode = MqttReasonCodes.Disconnect.SESSION_TAKEN_OVER; - break; - case SESSION_TIMEOUT: - returnCode = MqttReasonCodes.Disconnect.MAXIMUM_CONNECT_TIME; - break; - } + MqttReasonCodes.Disconnect returnCode = switch (sessionCloseNotification.getReason()) { + case CREDENTIALS_UPDATED, RPC_DELIVERY_TIMEOUT -> MqttReasonCodes.Disconnect.ADMINISTRATIVE_ACTION; + case MAX_CONCURRENT_SESSIONS_LIMIT_REACHED -> MqttReasonCodes.Disconnect.SESSION_TAKEN_OVER; + case SESSION_TIMEOUT -> MqttReasonCodes.Disconnect.MAXIMUM_CONNECT_TIME; + default -> MqttReasonCodes.Disconnect.IMPLEMENTATION_SPECIFIC_ERROR; + }; closeCtx(deviceSessionCtx.getChannel(), returnCode); } From 020d0182dae6a804dac524079b036d6c8b543740 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 4 Nov 2024 13:26:42 +0200 Subject: [PATCH 23/71] Sync postgres data in kafka case --- .../edge/rpc/AbstractEdgeGrpcSession.java | 131 +++++++++--------- .../service/edge/rpc/EdgeGrpcService.java | 12 +- .../edge/rpc/KafkaEdgeEventService.java | 7 +- .../edge/rpc/KafkaEdgeGrpcSession.java | 2 +- 4 files changed, 83 insertions(+), 69 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java index 9aa21dd265..0baa8a90a4 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java @@ -115,16 +115,16 @@ public abstract class AbstractEdgeGrpcSession highPriorityQueue = new ConcurrentLinkedQueue<>(); - private UUID sessionId; + protected UUID sessionId; private BiConsumer sessionOpenListener; private BiConsumer sessionCloseListener; private final EdgeSessionState sessionState = new EdgeSessionState(); private final ReentrantLock downlinkMsgLock = new ReentrantLock(); - private EdgeContextComponent ctx; - private Edge edge; - private TenantId tenantId; + protected EdgeContextComponent ctx; + protected Edge edge; + protected TenantId tenantId; private Long newStartTs; private Long previousStartTs; @@ -162,7 +162,7 @@ public abstract class AbstractEdgeGrpcSession() { + inputStream = new StreamObserver<>() { @Override public void onNext(RequestMsg requestMsg) { if (!connected && requestMsg.getMsgType().equals(RequestMsgType.CONNECT_RPC_MESSAGE)) { @@ -233,9 +233,9 @@ public abstract class AbstractEdgeGrpcSession> future = startProcessingEdgeEvents(next); Futures.addCallback(future, new FutureCallback<>() { @Override @@ -270,7 +270,7 @@ public abstract class AbstractEdgeGrpcSession pageData = fetcher.fetchEdgeEvents(edge.getTenantId(), edge, pageLink); if (isConnected() && !pageData.getData().isEmpty()) { - log.trace("[{}][{}][{}] event(s) are going to be processed.", this.tenantId, this.sessionId, pageData.getData().size()); + log.trace("[{}][{}][{}] event(s) are going to be processed.", tenantId, sessionId, pageData.getData().size()); List downlinkMsgsPack = convertToDownlinkMsgsPack(pageData.getData()); + for (DownlinkMsg downlinkMsg : downlinkMsgsPack) { + if (downlinkMsg.getEntityDataCount() > 0) { + System.out.println("downlink = " + downlinkMsg); + } + } Futures.addCallback(sendDownlinkMsgsPack(downlinkMsgsPack), new FutureCallback<>() { @Override public void onSuccess(@Nullable Boolean isInterrupted) { @@ -329,25 +334,25 @@ public abstract class AbstractEdgeGrpcSession optional = ctx.getEdgeService().findEdgeByRoutingKey(TenantId.SYS_TENANT_ID, request.getEdgeRoutingKey()); if (optional.isPresent()) { edge = optional.get(); - this.tenantId = edge.getTenantId(); + tenantId = edge.getTenantId(); try { if (edge.getSecret().equals(request.getEdgeSecret())) { sessionOpenListener.accept(edge.getId(), (T) this); - this.edgeVersion = request.getEdgeVersion(); + edgeVersion = request.getEdgeVersion(); processSaveEdgeVersionAsAttribute(request.getEdgeVersion().name()); return ConnectResponseMsg.newBuilder() .setResponseCode(ConnectResponseCode.ACCEPTED) @@ -383,11 +388,11 @@ public abstract class AbstractEdgeGrpcSession this.clientMaxInboundMessageSize) { + if (clientMaxInboundMessageSize != 0 && downlinkMsg.getSerializedSize() > clientMaxInboundMessageSize) { String error = String.format("Client max inbound message size %s is exceeded. Please increase value of CLOUD_RPC_MAX_INBOUND_MESSAGE_SIZE " + - "env variable on the edge and restart it.", this.clientMaxInboundMessageSize); + "env variable on the edge and restart it.", clientMaxInboundMessageSize); String message = String.format("Downlink msg size %s exceeds client max inbound message size %s. " + - "Please increase value of CLOUD_RPC_MAX_INBOUND_MESSAGE_SIZE env variable on the edge and restart it.", downlinkMsg.getSerializedSize(), this.clientMaxInboundMessageSize); - log.error("[{}][{}][{}] {} Message {}", this.tenantId, edge.getId(), this.sessionId, message, downlinkMsg); + "Please increase value of CLOUD_RPC_MAX_INBOUND_MESSAGE_SIZE env variable on the edge and restart it.", downlinkMsg.getSerializedSize(), clientMaxInboundMessageSize); + log.error("[{}][{}][{}] {} Message {}", tenantId, edge.getId(), sessionId, message, downlinkMsg); ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId) .edgeId(edge.getId()).customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(message).error(error).build()); sessionState.getPendingMsgsMap().remove(downlinkMsg.getDownlinkMsgId()); @@ -478,7 +483,7 @@ public abstract class AbstractEdgeGrpcSession downlinkMsgsPack = convertToDownlinkMsgsPack(highPriorityEvents); sendDownlinkMsgsPack(downlinkMsgsPack).get(); } catch (Exception e) { - log.error("[{}] Failed to process high priority events", this.sessionId, e); + log.error("[{}] Failed to process high priority events", sessionId, e); } } protected ListenableFuture processEdgeEvents() throws Exception { SettableFuture result = SettableFuture.create(); - log.trace("[{}][{}] starting processing edge events", this.tenantId, this.sessionId); + log.trace("[{}][{}] starting processing edge events", tenantId, sessionId); if (isConnected() && isSyncCompleted()) { Pair startTsAndSeqId = getQueueStartTsAndSeqId().get(); - this.previousStartTs = startTsAndSeqId.getFirst(); - this.previousStartSeqId = startTsAndSeqId.getSecond(); + previousStartTs = startTsAndSeqId.getFirst(); + previousStartSeqId = startTsAndSeqId.getSecond(); GeneralEdgeEventFetcher fetcher = new GeneralEdgeEventFetcher( - this.previousStartTs, - this.previousStartSeqId, - this.seqIdEnd, + previousStartTs, + previousStartSeqId, + seqIdEnd, false, Integer.toUnsignedLong(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount()), ctx.getEdgeEventService()); @@ -593,7 +598,7 @@ public abstract class AbstractEdgeGrpcSession convertToDownlinkMsgsPack(List edgeEvents) { List result = new ArrayList<>(); for (EdgeEvent edgeEvent : edgeEvents) { - log.trace("[{}][{}] converting edge event to downlink msg [{}]", this.tenantId, this.sessionId, edgeEvent); + log.trace("[{}][{}] converting edge event to downlink msg [{}]", tenantId, sessionId, edgeEvent); DownlinkMsg downlinkMsg = null; try { switch (edgeEvent.getAction()) { @@ -624,18 +629,18 @@ public abstract class AbstractEdgeGrpcSession { downlinkMsg = convertEntityEventToDownlink(edgeEvent); if (downlinkMsg != null && downlinkMsg.getWidgetTypeUpdateMsgCount() > 0) { - log.trace("[{}][{}] widgetTypeUpdateMsg message processed, downlinkMsgId = {}", this.tenantId, this.sessionId, downlinkMsg.getDownlinkMsgId()); + log.trace("[{}][{}] widgetTypeUpdateMsg message processed, downlinkMsgId = {}", tenantId, sessionId, downlinkMsg.getDownlinkMsgId()); } else { - log.trace("[{}][{}] entity message processed [{}]", this.tenantId, this.sessionId, downlinkMsg); + log.trace("[{}][{}] entity message processed [{}]", tenantId, sessionId, downlinkMsg); } } case ATTRIBUTES_UPDATED, POST_ATTRIBUTES, ATTRIBUTES_DELETED, TIMESERIES_UPDATED -> downlinkMsg = ctx.getTelemetryProcessor().convertTelemetryEventToDownlink(edge, edgeEvent); default -> - log.warn("[{}][{}] Unsupported action type [{}]", this.tenantId, this.sessionId, edgeEvent.getAction()); + log.warn("[{}][{}] Unsupported action type [{}]", tenantId, sessionId, edgeEvent.getAction()); } } catch (Exception e) { - log.error("[{}][{}] Exception during converting edge event to downlink msg", this.tenantId, this.sessionId, e); + log.error("[{}][{}] Exception during converting edge event to downlink msg", tenantId, sessionId, e); } if (downlinkMsg != null) { result.add(downlinkMsg); @@ -667,22 +672,22 @@ public abstract class AbstractEdgeGrpcSession edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), 0L, this.previousStartSeqId == 0 ? null : this.previousStartSeqId - 1, pageLink); + TimePageLink pageLink = new TimePageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount(), 0, null, null, newStartTs, System.currentTimeMillis()); + PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), 0L, previousStartSeqId == 0 ? null : previousStartSeqId - 1, pageLink); return !edgeEvents.getData().isEmpty(); } catch (Exception e) { - log.error("[{}][{}][{}] Failed to execute isSeqIdStartedNewCycle", this.tenantId, edge.getId(), sessionId, e); + log.error("[{}][{}][{}] Failed to execute isSeqIdStartedNewCycle", tenantId, edge.getId(), sessionId, e); } return false; } private boolean isNewEdgeEventsAvailable() { try { - TimePageLink pageLink = new TimePageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount(), 0, null, null, this.newStartTs, System.currentTimeMillis()); - PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), this.newStartSeqId, null, pageLink); + TimePageLink pageLink = new TimePageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount(), 0, null, null, newStartTs, System.currentTimeMillis()); + PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), newStartSeqId, null, pageLink); return !edgeEvents.getData().isEmpty() || !highPriorityQueue.isEmpty(); } catch (Exception e) { - log.error("[{}][{}][{}] Failed to execute isNewEdgeEventsAvailable", this.tenantId, edge.getId(), sessionId, e); + log.error("[{}][{}][{}] Failed to execute isNewEdgeEventsAvailable", tenantId, edge.getId(), sessionId, e); } return false; } @@ -696,18 +701,18 @@ public abstract class AbstractEdgeGrpcSession> updateQueueStartTsAndSeqId(Pair pair) { - this.newStartTs = pair.getFirst(); - this.newStartSeqId = pair.getSecond(); - log.trace("[{}] updateQueueStartTsAndSeqId [{}][{}][{}]", this.sessionId, edge.getId(), this.newStartTs, this.newStartSeqId); + newStartTs = pair.getFirst(); + newStartSeqId = pair.getSecond(); + log.trace("[{}] updateQueueStartTsAndSeqId [{}][{}][{}]", sessionId, edge.getId(), newStartTs, newStartSeqId); List attributes = Arrays.asList( - new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_TS_ATTR_KEY, this.newStartTs), System.currentTimeMillis()), - new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_SEQ_ID_ATTR_KEY, this.newStartSeqId), System.currentTimeMillis())); + new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_TS_ATTR_KEY, newStartTs), System.currentTimeMillis()), + new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_SEQ_ID_ATTR_KEY, newStartSeqId), System.currentTimeMillis())); return ctx.getAttributesService().save(edge.getTenantId(), edge.getId(), AttributeScope.SERVER_SCOPE, attributes); } @@ -734,22 +739,22 @@ public abstract class AbstractEdgeGrpcSession 0) { - log.trace("[{}][{}] Sending downlink widgetTypeUpdateMsg, downlinkMsgId = {}", this.tenantId, this.sessionId, downlinkMsg.getDownlinkMsg().getDownlinkMsgId()); + log.trace("[{}][{}] Sending downlink widgetTypeUpdateMsg, downlinkMsgId = {}", tenantId, sessionId, downlinkMsg.getDownlinkMsg().getDownlinkMsgId()); } else { - log.trace("[{}][{}] Sending downlink msg [{}]", this.tenantId, this.sessionId, downlinkMsg); + log.trace("[{}][{}] Sending downlink msg [{}]", tenantId, sessionId, downlinkMsg); } if (isConnected()) { downlinkMsgLock.lock(); try { outputStream.onNext(downlinkMsg); } catch (Exception e) { - log.error("[{}][{}] Failed to send downlink message [{}]", this.tenantId, this.sessionId, downlinkMsg, e); + log.error("[{}][{}] Failed to send downlink message [{}]", tenantId, sessionId, downlinkMsg, e); connected = false; sessionCloseListener.accept(edge, sessionId); } finally { downlinkMsgLock.unlock(); } - log.trace("[{}][{}] Response msg successfully sent. downlinkMsgId = {}", this.tenantId, this.sessionId, downlinkMsg.getDownlinkMsg().getDownlinkMsgId()); + log.trace("[{}][{}] Response msg successfully sent. downlinkMsgId = {}", tenantId, sessionId, downlinkMsg.getDownlinkMsg().getDownlinkMsgId()); } } @@ -805,7 +810,7 @@ public abstract class AbstractEdgeGrpcSession session) { EdgeId edgeId = session.getEdge().getId(); - UUID tenantId = session.getEdge().getTenantId().getId(); + TenantId tenantId = session.getEdge().getTenantId(); if (sessions.containsKey(edgeId)) { ScheduledFuture edgeEventCheckTask = edgeEventProcessingExecutorService.schedule(() -> { try { @@ -423,8 +422,10 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i if (Boolean.TRUE.equals(newEventsAdded)) { sessionNewEvents.put(edgeId, true); } - if (isKafkaSupported) { + if (session instanceof KafkaEdgeGrpcSession kafkaEdgeGrpcSession && newEventsAdded != null) { edgeEventsProcessed.put(edgeId, true); + initializeKafkaConsumer(kafkaEdgeGrpcSession, tenantId, edgeId); + cancelScheduleEdgeEventsCheck(edgeId); } else { scheduleEdgeEventsCheck(session); } @@ -437,7 +438,10 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } }, ctx.getGrpcCallbackExecutorService()); } else { - if (!isKafkaSupported) { + if (Boolean.TRUE.equals(edgeEventsProcessed.get(edgeId)) && session instanceof KafkaEdgeGrpcSession kafkaEdgeGrpcSession) { + initializeKafkaConsumer(kafkaEdgeGrpcSession, tenantId, edgeId); + cancelScheduleEdgeEventsCheck(edgeId); + } else { scheduleEdgeEventsCheck(session); } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java index 77760524a5..1126214e60 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.dao.edge.EdgeEventDao; import org.thingsboard.server.dao.edge.EdgeEventService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -54,6 +55,7 @@ public class KafkaEdgeEventService implements EdgeEventService { private final TbQueueProducerProvider producerProvider; @Lazy private final TopicService topicService; + private final EdgeEventDao edgeEventDao; @Override public ListenableFuture saveAsync(EdgeEvent edgeEvent) { @@ -74,11 +76,14 @@ public class KafkaEdgeEventService implements EdgeEventService { @Override public PageData findEdgeEvents(TenantId tenantId, EdgeId edgeId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink) { - return null; + // To support fetching edge events on connect from postgres if there are any: + return edgeEventDao.findEdgeEvents(tenantId.getId(), edgeId, seqIdStart, seqIdEnd, pageLink); } @Override public void cleanupEvents(long ttl) { + // To delete deprecated events by ttl + edgeEventDao.cleanupEvents(ttl); } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java index b86aa0ecf2..920047898b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java @@ -71,7 +71,7 @@ public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession> msgs, TbQueueConsumer> consumer) { - log.trace("[{}][{}] starting processing edge events", this.tenantId, this.sessionId); + log.trace("[{}][{}] starting processing edge events", tenantId, sessionId); if (isConnected() && isSyncCompleted()) { if (!highPriorityQueue.isEmpty()) { From c480790e6340ff2cfbe6612ababdbc69080fb5d8 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Mon, 4 Nov 2024 16:26:15 +0200 Subject: [PATCH 24/71] Added tests for session close on RPC delivery timeout --- ...onOnRpcDeliveryTimeoutIntegrationTest.java | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv5/rpc/MqttV5CloseTransportSessionOnRpcDeliveryTimeoutIntegrationTest.java diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv5/rpc/MqttV5CloseTransportSessionOnRpcDeliveryTimeoutIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv5/rpc/MqttV5CloseTransportSessionOnRpcDeliveryTimeoutIntegrationTest.java new file mode 100644 index 0000000000..dc366c11f2 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv5/rpc/MqttV5CloseTransportSessionOnRpcDeliveryTimeoutIntegrationTest.java @@ -0,0 +1,157 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.mqtt.mqttv5.rpc; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.NullNode; +import io.netty.handler.codec.mqtt.MqttQoS; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.eclipse.paho.mqttv5.client.IMqttToken; +import org.eclipse.paho.mqttv5.client.MqttDisconnectResponse; +import org.eclipse.paho.mqttv5.common.MqttException; +import org.eclipse.paho.mqttv5.common.MqttMessage; +import org.eclipse.paho.mqttv5.common.packet.MqttProperties; +import org.eclipse.paho.mqttv5.common.packet.MqttReturnCode; +import org.eclipse.paho.mqttv5.common.packet.MqttWireMessage; +import org.junit.Before; +import org.junit.Test; +import org.springframework.test.context.TestPropertySource; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.rpc.Rpc; +import org.thingsboard.server.common.data.rpc.RpcStatus; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.transport.mqtt.MqttTestConfigProperties; +import org.thingsboard.server.transport.mqtt.mqttv5.MqttV5TestCallback; +import org.thingsboard.server.transport.mqtt.mqttv5.MqttV5TestClient; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.common.data.device.profile.MqttTopics.DEVICE_RPC_REQUESTS_SUB_TOPIC; + +@Slf4j +@DaoSqlTest +@TestPropertySource(properties = { + "actors.rpc.close_session_on_rpc_delivery_timeout=true", + "transport.mqtt.timeout=100", +}) +public class MqttV5CloseTransportSessionOnRpcDeliveryTimeoutIntegrationTest extends AbstractMqttV5RpcTest { + + @Before + public void beforeTest() throws Exception { + MqttTestConfigProperties configProperties = MqttTestConfigProperties.builder() + .deviceName("RPC test device") + .build(); + processBeforeTest(configProperties); + } + + @Test + public void testOneWayRpcCloseSessionOnRpcDeliveryTimeout() throws Exception { + testCloseSessionOnRpcDeliveryTimeout("oneway"); + } + + @Test + public void testTwoWayRpcCloseSessionOnRpcDeliveryTimeout() throws Exception { + testCloseSessionOnRpcDeliveryTimeout("twoway"); + } + + private void testCloseSessionOnRpcDeliveryTimeout(String rpcType) throws Exception { + MqttV5TestClient client = new MqttV5TestClient(); + client.enableManualAcks(); + client.connectAndWait(accessToken); + MqttV5NoAckTestCallback callback = new MqttV5NoAckTestCallback(); + client.setCallback(callback); + client.subscribeAndWait(DEVICE_RPC_REQUESTS_SUB_TOPIC, MqttQoS.AT_LEAST_ONCE); + + String expectedReceivedPayload = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; + long expirationTime = System.currentTimeMillis() + TimeUnit.DAYS.toMillis(1); + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"persistent\":true,\"retries\":0,\"expirationTime\": " + expirationTime + "}"; + String result = doPostAsync("/api/rpc/" + rpcType + "/" + savedDevice.getId(), setGpioRequest, String.class, status().isOk()); + + assertThat(result).isNotNull(); + JsonNode response = JacksonUtil.toJsonNode(result); + assertThat(response).isNotNull(); + assertThat(response.hasNonNull("rpcId")).isTrue(); + + callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(callback.getQoS()).isEqualTo(MqttQoS.AT_LEAST_ONCE.value()); + assertThat(JacksonUtil.fromBytes(callback.getPayloadBytes())) + .isEqualTo(JacksonUtil.toJsonNode(expectedReceivedPayload)); + + callback.getDisconnectLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(callback.getReturnCode()).isEqualTo(MqttReturnCode.RETURN_CODE_ADMINISTRITIVE_ACTION); + + Rpc persistedRpc = doGet("/api/rpc/persistent/" + response.get("rpcId").asText(), Rpc.class); + assertThat(persistedRpc).isNotNull(); + assertThat(persistedRpc.getStatus()).isEqualTo(RpcStatus.QUEUED); + assertThat(persistedRpc.getResponse()).isInstanceOf(NullNode.class); + assertThat(client.isConnected()).isFalse(); + } + + @Getter + private static class MqttV5NoAckTestCallback extends MqttV5TestCallback { + + private int returnCode; + + public CountDownLatch getDisconnectLatch() { + return super.getDeliveryLatch(); + } + + @Override + public void disconnected(MqttDisconnectResponse mqttDisconnectResponse) { + log.warn("MqttDisconnectResponse: {}", mqttDisconnectResponse); + returnCode = mqttDisconnectResponse.getReturnCode(); + getDisconnectLatch().countDown(); + } + + @Override + public void mqttErrorOccurred(MqttException e) { + log.warn("Error occurred:", e); + } + + @Override + public void messageArrived(String requestTopic, MqttMessage mqttMessage) { + log.warn("messageArrived on topic: {}", requestTopic); + qoS = mqttMessage.getQos(); + payloadBytes = mqttMessage.getPayload(); + subscribeLatch.countDown(); + } + + @Override + public void deliveryComplete(IMqttToken iMqttToken) { + // should be never called, Since we're never going to send a response back to server. + log.warn("delivery complete: {}", iMqttToken.getResponse()); + pubAckReceived = iMqttToken.getResponse().getType() == MqttWireMessage.MESSAGE_TYPE_PUBACK; + getDisconnectLatch().countDown(); + } + + @Override + public void connectComplete(boolean reconnect, String serverURI) { + log.warn("Connect completed: reconnect - {}, serverURI - {}", reconnect, serverURI); + } + + @Override + public void authPacketArrived(int reasonCode, MqttProperties mqttProperties) { + log.warn("Auth package received: reasonCode - {}, mqtt properties - {}", reasonCode, mqttProperties); + } + + + } + +} From a6c8b9a5264fba8e93fc258c2b02432206286f04 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Tue, 5 Nov 2024 17:35:19 +0200 Subject: [PATCH 25/71] Fix queue.proto structure of EdgeEventMsg --- .../edge/rpc/processor/BaseEdgeProcessor.java | 18 ++++++------------ .../server/common/util/ProtoUtils.java | 8 +++++++- common/proto/src/main/proto/queue.proto | 8 +++++--- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index eb6b0bf47a..e8e2ca7c3a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -119,8 +119,7 @@ public abstract class BaseEdgeProcessor { UPDATED_COMMENT -> true; default -> switch (type) { case ALARM, ALARM_COMMENT, RULE_CHAIN, RULE_CHAIN_METADATA, USER, CUSTOMER, TENANT, TENANT_PROFILE, - WIDGETS_BUNDLE, WIDGET_TYPE, - ADMIN_SETTINGS, OTA_PACKAGE, QUEUE, RELATION, NOTIFICATION_TEMPLATE, NOTIFICATION_TARGET, + WIDGETS_BUNDLE, WIDGET_TYPE, ADMIN_SETTINGS, OTA_PACKAGE, QUEUE, RELATION, NOTIFICATION_TEMPLATE, NOTIFICATION_TARGET, NOTIFICATION_RULE -> true; default -> false; }; @@ -176,10 +175,8 @@ public abstract class BaseEdgeProcessor { return switch (actionType) { case UPDATED, CREDENTIALS_UPDATED, ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER, UPDATED_COMMENT -> UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE; - case ADDED, ASSIGNED_TO_EDGE, RELATION_ADD_OR_UPDATE, ADDED_COMMENT -> - UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE; - case DELETED, UNASSIGNED_FROM_EDGE, RELATION_DELETED, DELETED_COMMENT, ALARM_DELETE -> - UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE; + case ADDED, ASSIGNED_TO_EDGE, RELATION_ADD_OR_UPDATE, ADDED_COMMENT -> UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE; + case DELETED, UNASSIGNED_FROM_EDGE, RELATION_DELETED, DELETED_COMMENT, ALARM_DELETE -> UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE; case ALARM_ACK -> UpdateMsgType.ALARM_ACK_RPC_MESSAGE; case ALARM_CLEAR -> UpdateMsgType.ALARM_CLEAR_RPC_MESSAGE; default -> throw new RuntimeException("Unsupported actionType [" + actionType + "]"); @@ -322,13 +319,10 @@ public abstract class BaseEdgeProcessor { case TENANT -> edgeCtx.getTenantService().findTenantById(tenantId) != null; case DEVICE -> edgeCtx.getDeviceService().findDeviceById(tenantId, new DeviceId(entityId.getId())) != null; case ASSET -> edgeCtx.getAssetService().findAssetById(tenantId, new AssetId(entityId.getId())) != null; - case ENTITY_VIEW -> - edgeCtx.getEntityViewService().findEntityViewById(tenantId, new EntityViewId(entityId.getId())) != null; - case CUSTOMER -> - edgeCtx.getCustomerService().findCustomerById(tenantId, new CustomerId(entityId.getId())) != null; + case ENTITY_VIEW -> edgeCtx.getEntityViewService().findEntityViewById(tenantId, new EntityViewId(entityId.getId())) != null; + case CUSTOMER -> edgeCtx.getCustomerService().findCustomerById(tenantId, new CustomerId(entityId.getId())) != null; case USER -> edgeCtx.getUserService().findUserById(tenantId, new UserId(entityId.getId())) != null; - case DASHBOARD -> - edgeCtx.getDashboardService().findDashboardById(tenantId, new DashboardId(entityId.getId())) != null; + case DASHBOARD -> edgeCtx.getDashboardService().findDashboardById(tenantId, new DashboardId(entityId.getId())) != null; case EDGE -> edgeCtx.getEdgeService().findEdgeById(tenantId, new EdgeId(entityId.getId())) != null; default -> false; }; diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 405371d27a..264b23f118 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -190,6 +190,10 @@ public class ProtoUtils { builder.setEntityType(edgeEvent.getType().name()); builder.setAction(edgeEvent.getAction().name()); + if (edgeEvent.getEdgeId() != null) { + builder.setEdgeIdMSB(edgeEvent.getEdgeId().getId().getMostSignificantBits()); + builder.setEdgeIdLSB(edgeEvent.getEdgeId().getId().getLeastSignificantBits()); + } if (edgeEvent.getEntityId() != null) { builder.setEntityIdMSB(edgeEvent.getEntityId().getMostSignificantBits()); builder.setEntityIdLSB(edgeEvent.getEntityId().getLeastSignificantBits()); @@ -208,10 +212,12 @@ public class ProtoUtils { edgeEvent.setType(EdgeEventType.valueOf(proto.getEntityType())); edgeEvent.setAction(EdgeEventActionType.valueOf(proto.getAction())); + if (proto.hasEdgeIdMSB() && proto.hasEdgeIdLSB()) { + edgeEvent.setEdgeId(new EdgeId(new UUID(proto.getEdgeIdMSB(), proto.getEdgeIdLSB()))); + } if (proto.hasEntityIdMSB() && proto.hasEntityIdLSB()) { edgeEvent.setEntityId(new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); } - if (proto.hasBody()) { edgeEvent.setBody(JacksonUtil.toJsonNode(proto.getBody())); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 5d3ce2a272..c48677b8a6 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1131,9 +1131,11 @@ message EdgeEventMsgProto { int64 tenantIdLSB = 2; string entityType = 3; string action = 4; - optional int64 entityIdMSB = 5; - optional int64 entityIdLSB = 6; - optional string body = 7; + optional int64 edgeIdMSB = 5; + optional int64 edgeIdLSB = 6; + optional int64 entityIdMSB = 7; + optional int64 entityIdLSB = 8; + optional string body = 9; } message EdgeNotificationMsgProto { From 78f7dd3dcd9f790a2b93e0133c230ff0528548a9 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Wed, 6 Nov 2024 11:42:05 +0200 Subject: [PATCH 26/71] Minor fix of thingsboard.yml config value --- application/src/main/resources/thingsboard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index bf050d268a..e172f77a69 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1521,7 +1521,7 @@ queue: # Example of specific consumer properties value per topic for edge event - key: max.poll.records # Example of specific consumer properties value per topic for edge event - value: "${TB_QUEUE_KAFKA_EDGE_EVENT_MAX_POLL_INTERVAL_MS:50}" + value: "${TB_QUEUE_KAFKA_EDGE_EVENT_MAX_POLL_RECORDS:50}" tb_housekeeper: # Consumer properties for Housekeeper tasks topic - key: max.poll.records From b0a5a97bab21ebf90ce04c0c9093024a0c15cd5c Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 6 Nov 2024 15:24:02 +0100 Subject: [PATCH 27/71] minor refactoring due to comments --- .../actors/ruleChain/DefaultTbContext.java | 37 +++++++------------ ...AbstractRuleEngineFlowIntegrationTest.java | 2 +- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 23741ad965..5b4902afa5 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -110,7 +110,6 @@ import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.common.SimpleTbQueueCallback; import org.thingsboard.server.service.executors.PubSubRuleNodeExecutorProvider; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; @@ -177,24 +176,10 @@ class DefaultTbContext implements TbContext { if (!msg.isValid()) { return; } - TbMsg inputMsg = msg.copyWithRuleChainId(ruleChainId); - inputMsg.pushToStack(nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId()); - TransportProtos.ToRuleEngineMsg toReMsg = TransportProtos.ToRuleEngineMsg.newBuilder() - .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(inputMsg)).build(); - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getQueueName(), getTenantId(), inputMsg.getOriginator()); - mainCtx.getClusterService().pushMsgToRuleEngine(tpi, inputMsg.getId(), toReMsg, new TbQueueCallback() { - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - ack(msg); - } - - @Override - public void onFailure(Throwable error) { - tellFailure(msg, error); - } - }); + TbMsg tbMsg = msg.copyWithRuleChainId(ruleChainId); + tbMsg.pushToStack(nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId()); + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getQueueName(), getTenantId(), tbMsg.getOriginator()); + doEnqueue(tpi, tbMsg, new SimpleTbQueueCallback(md -> ack(msg), t -> tellFailure(msg, t))); } @Override @@ -230,14 +215,10 @@ class DefaultTbContext implements TbContext { } return; } - TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() - .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(tbMsg)).build(); if (nodeCtx.getSelf().isDebugMode()) { mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), tbMsg, "To Root Rule Chain"); } - mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, new SimpleTbQueueCallback( + doEnqueue(tpi, tbMsg, new SimpleTbQueueCallback( metadata -> { if (onSuccess != null) { onSuccess.run(); @@ -252,6 +233,14 @@ class DefaultTbContext implements TbContext { })); } + private void doEnqueue(TopicPartitionInfo tpi, TbMsg tbMsg, TbQueueCallback callback) { + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, callback); + } + @Override public void enqueueForTellFailure(TbMsg tbMsg, String failureMessage) { TopicPartitionInfo tpi = resolvePartition(tbMsg); diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 920ffdd211..143c6c72a1 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -333,7 +333,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule RuleChain finalRuleChain = rootRuleChain; RuleNode lastRuleNode = secondaryMetaData.getNodes().stream().filter(node -> !node.getId().equals(finalRuleChain.getFirstRuleNodeId())).findFirst().get(); - Awaitility.await().atMost(3, TimeUnit.SECONDS) + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getDebugEvents(savedTenant.getId(), lastRuleNode.getId(), 1000) .getData() From f5c0ae68a6bea9a20bd69ef2eeabc54af8c71ce6 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Thu, 7 Nov 2024 11:52:36 +0200 Subject: [PATCH 28/71] Add scheduler to read hpqueue for session --- .../edge/rpc/AbstractEdgeGrpcSession.java | 39 ++++++---------- .../service/edge/rpc/EdgeGrpcService.java | 11 ++++- .../edge/rpc/KafkaEdgeGrpcSession.java | 45 ++++++++++++------- 3 files changed, 50 insertions(+), 45 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java index 0baa8a90a4..15ee44a66e 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java @@ -623,10 +623,8 @@ public abstract class AbstractEdgeGrpcSession { + ALARM_DELETE, CREDENTIALS_UPDATED, RELATION_ADD_OR_UPDATE, RELATION_DELETED, RPC_CALL, + ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER, ADDED_COMMENT, UPDATED_COMMENT, DELETED_COMMENT -> { downlinkMsg = convertEntityEventToDownlink(edgeEvent); if (downlinkMsg != null && downlinkMsg.getWidgetTypeUpdateMsgCount() > 0) { log.trace("[{}][{}] widgetTypeUpdateMsg message processed, downlinkMsgId = {}", tenantId, sessionId, downlinkMsg.getDownlinkMsgId()); @@ -636,8 +634,7 @@ public abstract class AbstractEdgeGrpcSession downlinkMsg = ctx.getTelemetryProcessor().convertTelemetryEventToDownlink(edge, edgeEvent); - default -> - log.warn("[{}][{}] Unsupported action type [{}]", tenantId, sessionId, edgeEvent.getAction()); + default -> log.warn("[{}][{}] Unsupported action type [{}]", tenantId, sessionId, edgeEvent.getAction()); } } catch (Exception e) { log.error("[{}][{}] Exception during converting edge event to downlink msg", tenantId, sessionId, e); @@ -763,40 +760,30 @@ public abstract class AbstractEdgeGrpcSession ctx.getEdgeProcessor().convertEdgeEventToDownlink(edgeEvent); case DEVICE -> ctx.getDeviceProcessor().convertDeviceEventToDownlink(edgeEvent, edgeVersion); - case DEVICE_PROFILE -> - ctx.getDeviceProfileProcessor().convertDeviceProfileEventToDownlink(edgeEvent, edgeVersion); - case ASSET_PROFILE -> - ctx.getAssetProfileProcessor().convertAssetProfileEventToDownlink(edgeEvent, edgeVersion); - case ASSET -> - ctx.getAssetProcessor().convertAssetEventToDownlink(edgeEvent, edgeVersion); + case DEVICE_PROFILE -> ctx.getDeviceProfileProcessor().convertDeviceProfileEventToDownlink(edgeEvent, edgeVersion); + case ASSET_PROFILE -> ctx.getAssetProfileProcessor().convertAssetProfileEventToDownlink(edgeEvent, edgeVersion); + case ASSET -> ctx.getAssetProcessor().convertAssetEventToDownlink(edgeEvent, edgeVersion); case ENTITY_VIEW -> ctx.getEntityViewProcessor().convertEntityViewEventToDownlink(edgeEvent, edgeVersion); case DASHBOARD -> ctx.getDashboardProcessor().convertDashboardEventToDownlink(edgeEvent, edgeVersion); case CUSTOMER -> ctx.getCustomerProcessor().convertCustomerEventToDownlink(edgeEvent, edgeVersion); case RULE_CHAIN -> ctx.getRuleChainProcessor().convertRuleChainEventToDownlink(edgeEvent, edgeVersion); - case RULE_CHAIN_METADATA -> - ctx.getRuleChainProcessor().convertRuleChainMetadataEventToDownlink(edgeEvent, edgeVersion); + case RULE_CHAIN_METADATA -> ctx.getRuleChainProcessor().convertRuleChainMetadataEventToDownlink(edgeEvent, edgeVersion); case ALARM -> ctx.getAlarmProcessor().convertAlarmEventToDownlink(edgeEvent, edgeVersion); case ALARM_COMMENT -> ctx.getAlarmProcessor().convertAlarmCommentEventToDownlink(edgeEvent, edgeVersion); case USER -> ctx.getUserProcessor().convertUserEventToDownlink(edgeEvent, edgeVersion); case RELATION -> ctx.getRelationProcessor().convertRelationEventToDownlink(edgeEvent, edgeVersion); - case WIDGETS_BUNDLE -> - ctx.getWidgetBundleProcessor().convertWidgetsBundleEventToDownlink(edgeEvent, edgeVersion); + case WIDGETS_BUNDLE -> ctx.getWidgetBundleProcessor().convertWidgetsBundleEventToDownlink(edgeEvent, edgeVersion); case WIDGET_TYPE -> ctx.getWidgetTypeProcessor().convertWidgetTypeEventToDownlink(edgeEvent, edgeVersion); - case ADMIN_SETTINGS -> - ctx.getAdminSettingsProcessor().convertAdminSettingsEventToDownlink(edgeEvent, edgeVersion); + case ADMIN_SETTINGS -> ctx.getAdminSettingsProcessor().convertAdminSettingsEventToDownlink(edgeEvent, edgeVersion); case OTA_PACKAGE -> ctx.getOtaPackageProcessor().convertOtaPackageEventToDownlink(edgeEvent, edgeVersion); case TB_RESOURCE -> ctx.getResourceProcessor().convertResourceEventToDownlink(edgeEvent, edgeVersion); case QUEUE -> ctx.getQueueProcessor().convertQueueEventToDownlink(edgeEvent, edgeVersion); case TENANT -> ctx.getTenantProcessor().convertTenantEventToDownlink(edgeEvent, edgeVersion); - case TENANT_PROFILE -> - ctx.getTenantProfileProcessor().convertTenantProfileEventToDownlink(edgeEvent, edgeVersion); + case TENANT_PROFILE -> ctx.getTenantProfileProcessor().convertTenantProfileEventToDownlink(edgeEvent, edgeVersion); case NOTIFICATION_RULE -> ctx.getNotificationEdgeProcessor().convertNotificationRuleToDownlink(edgeEvent); - case NOTIFICATION_TARGET -> - ctx.getNotificationEdgeProcessor().convertNotificationTargetToDownlink(edgeEvent); - case NOTIFICATION_TEMPLATE -> - ctx.getNotificationEdgeProcessor().convertNotificationTemplateToDownlink(edgeEvent); - case OAUTH2_CLIENT -> - ctx.getOAuth2EdgeProcessor().convertOAuth2ClientEventToDownlink(edgeEvent, edgeVersion); + case NOTIFICATION_TARGET -> ctx.getNotificationEdgeProcessor().convertNotificationTargetToDownlink(edgeEvent); + case NOTIFICATION_TEMPLATE -> ctx.getNotificationEdgeProcessor().convertNotificationTemplateToDownlink(edgeEvent); + case OAUTH2_CLIENT -> ctx.getOAuth2EdgeProcessor().convertOAuth2ClientEventToDownlink(edgeEvent, edgeVersion); case DOMAIN -> ctx.getOAuth2EdgeProcessor().convertOAuth2DomainEventToDownlink(edgeEvent, edgeVersion); default -> { log.warn("[{}] Unsupported edge event type [{}]", edgeEvent.getTenantId(), edgeEvent); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java index 416445cc74..a82ba3efb2 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java @@ -95,6 +95,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i private final ConcurrentMap> sessionEdgeEventChecks = new ConcurrentHashMap<>(); private final ConcurrentMap> localSyncEdgeRequests = new ConcurrentHashMap<>(); private final ConcurrentMap edgeEventsProcessed = new ConcurrentHashMap<>(); + private final ConcurrentMap kafkaConsumerInit = new ConcurrentHashMap<>(); @Value("${edges.rpc.port}") private int rpcPort; @@ -336,6 +337,8 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i Boolean isChecked = edgeEventsProcessed.get(edgeId); if (Boolean.FALSE.equals(isChecked)) { scheduleEdgeEventsCheck(session); + } else { + initializeKafkaConsumer(session, tenantId, edgeId); } } if (edgeGrpcSession instanceof PostgresEdgeGrpcSession) { @@ -345,8 +348,11 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i private void initializeKafkaConsumer(KafkaEdgeGrpcSession kafkaEdgeGrpcSession, TenantId tenantId, EdgeId edgeId) { TbQueueConsumer> consumer = tbCoreQueueFactory.createEdgeEventMsgConsumer(tenantId, edgeId); - kafkaEdgeGrpcSession.initConsumer(() -> consumer, schedulerPoolSize); - kafkaEdgeGrpcSession.startConsumers(); + if (!kafkaConsumerInit.getOrDefault(edgeId, Boolean.FALSE)) { + kafkaEdgeGrpcSession.initConsumer(() -> consumer, schedulerPoolSize); + kafkaEdgeGrpcSession.startConsumers(); + kafkaConsumerInit.put(edgeId, Boolean.TRUE); + } } private void startSyncProcess(TenantId tenantId, EdgeId edgeId, UUID requestId, String requestServiceId) { @@ -486,6 +492,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } if (isKafkaSupported) { ((KafkaEdgeGrpcSession) toRemove).stopConsumer(); + kafkaConsumerInit.remove(edgeId); } TenantId tenantId = toRemove.getEdge().getTenantId(); save(tenantId, edgeId, DefaultDeviceStateService.ACTIVITY_STATE, false); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java index 920047898b..e16c89c23e 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java @@ -15,12 +15,9 @@ */ package org.thingsboard.server.service.edge.rpc; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; import io.grpc.stub.StreamObserver; import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; -import org.checkerframework.checker.nullness.qual.Nullable; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; @@ -40,6 +37,7 @@ import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.function.Supplier; @@ -47,6 +45,7 @@ import java.util.function.Supplier; public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession { private ExecutorService consumerExecutor; + private ScheduledExecutorService highPriorityExecutorService; private QueueConsumerManager> consumer; @@ -60,14 +59,16 @@ public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession>> edgeEventsConsumer, long schedulerPoolSize) { this.consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("edge-event-consumer")); + this.highPriorityExecutorService = Executors.newScheduledThreadPool((int) schedulerPoolSize, ThingsBoardThreadFactory.forName("edge-event-high-priority-scheduler")); this.consumer = QueueConsumerManager.>builder() .name("TB Edge events") .msgPackProcessor(this::processMsgs) - .pollInterval(schedulerPoolSize) + .pollInterval(ctx.getEdgeEventStorageSettings().getNoRecordsSleepInterval()) .consumerCreator(edgeEventsConsumer) .consumerExecutor(consumerExecutor) .threadPrefix("edge-events") .build(); + scheduleCheckForHighPriorityEvent(); } private void processMsgs(List> msgs, TbQueueConsumer> consumer) { @@ -83,26 +84,36 @@ public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession downlinkMsgsPack = convertToDownlinkMsgsPack(edgeEvents); - Futures.addCallback(sendDownlinkMsgsPack(downlinkMsgsPack), new FutureCallback<>() { - @Override - public void onSuccess(@Nullable Boolean isInterrupted) { - if (Boolean.TRUE.equals(isInterrupted)) { - log.debug("[{}][{}][{}] Send downlink messages task was interrupted", tenantId, edge.getId(), sessionId); - } else { - consumerExecutor.submit(consumer::commit); - } + try { + boolean isInterrupted = sendDownlinkMsgsPack(downlinkMsgsPack).get(); + if (isInterrupted) { + log.debug("[{}][{}][{}] Send downlink messages task was interrupted", tenantId, edge.getId(), sessionId); + } else { + consumer.commit(); } - @Override - public void onFailure(Throwable t) { - log.error("[{}] Failed to send downlink msgs pack", sessionId, t); - } - }, ctx.getGrpcCallbackExecutorService()); + } catch (Exception e) { + log.error("[{}] Failed to process all downlink messages", sessionId, e); + } } } else { log.trace("[{}][{}] edge is not connected or sync is not completed. Skipping iteration", tenantId, sessionId); } } + private void scheduleCheckForHighPriorityEvent() { + highPriorityExecutorService.scheduleAtFixedRate(() -> { + try { + if (isConnected() && isSyncCompleted()) { + if (!highPriorityQueue.isEmpty()) { + processHighPriorityEvents(); + } + } + } catch (Exception e) { + log.error("Error in processing high priority events", e); + } + }, 0, ctx.getEdgeEventStorageSettings().getNoRecordsSleepInterval() * 3, TimeUnit.MILLISECONDS); + } + public void startConsumers() { consumer.subscribe(); consumer.launch(); From 5535f11f224473de1326768a919b486a5558129f Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 12 Nov 2024 11:51:15 +0200 Subject: [PATCH 29/71] fixed log printing --- .../server/service/entitiy/user/DefaultUserService.java | 4 +--- .../java/org/thingsboard/server/dao/user/UserService.java | 2 +- .../java/org/thingsboard/server/dao/user/UserServiceImpl.java | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/user/DefaultUserService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/user/DefaultUserService.java index 3fe8f27873..f98f05b447 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/user/DefaultUserService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/user/DefaultUserService.java @@ -35,8 +35,6 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import org.thingsboard.server.service.security.system.SystemSecurityService; -import java.util.concurrent.TimeUnit; - @Service @TbCoreComponent @AllArgsConstructor @@ -90,7 +88,7 @@ public class DefaultUserService extends AbstractTbEntityService implements TbUse public UserActivationLink getActivationLink(TenantId tenantId, CustomerId customerId, UserId userId, HttpServletRequest request) throws ThingsboardException { UserCredentials userCredentials = userService.findUserCredentialsByUserId(tenantId, userId); if (!userCredentials.isEnabled() && userCredentials.getActivateToken() != null) { - userCredentials = userService.refreshUserActivationToken(tenantId, userCredentials); + userCredentials = userService.checkUserActivationToken(tenantId, userCredentials); String baseUrl = systemSecurityService.getBaseUrl(tenantId, customerId, request); String link = baseUrl + "/api/noauth/activate?activateToken=" + userCredentials.getActivateToken(); return new UserActivationLink(link, userCredentials.getActivationTokenTtl()); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java index 0d532cdfac..a33d64b746 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java @@ -63,7 +63,7 @@ public interface UserService extends EntityDaoService { UserCredentials generateUserActivationToken(UserCredentials userCredentials); - UserCredentials refreshUserActivationToken(TenantId tenantId, UserCredentials userCredentials); + UserCredentials checkUserActivationToken(TenantId tenantId, UserCredentials userCredentials); UserCredentials replaceUserCredentials(TenantId tenantId, UserCredentials userCredentials); diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index d8eee2967a..16b25a7ede 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -301,12 +301,12 @@ public class UserServiceImpl extends AbstractCachedEntityService Date: Tue, 12 Nov 2024 13:20:36 +0200 Subject: [PATCH 30/71] Refactoring GrpsSession --- .../edge/EdgeEventSourcingListener.java | 34 +---- .../edge/rpc/AbstractEdgeGrpcSession.java | 6 +- .../service/edge/rpc/EdgeGrpcService.java | 62 ++++----- .../service/edge/rpc/EdgeGrpcSession.java | 7 + .../edge/rpc/KafkaEdgeGrpcSession.java | 120 +++++++++--------- .../edge/rpc/PostgresEdgeGrpcSession.java | 7 + .../entitiy/EntityStateSourcingListener.java | 37 +++++- 7 files changed, 134 insertions(+), 139 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index 3653eb3a40..9317c51a64 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -20,10 +20,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.common.util.JacksonUtil; @@ -41,7 +37,6 @@ import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; -import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -55,10 +50,6 @@ import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.kafka.TbKafkaAdmin; -import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; /** * This event listener does not support async event processing because relay on ThreadLocal @@ -79,22 +70,11 @@ import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; @RequiredArgsConstructor public class EdgeEventSourcingListener { - private final TopicService topicService; private final TbClusterService tbClusterService; private final TenantService tenantService; private final EdgeSynchronizationManager edgeSynchronizationManager; - @Autowired(required = false) - @Lazy - private TbKafkaSettings kafkaSettings; - @Autowired(required = false) - @Lazy - private TbKafkaTopicConfigs kafkaTopicConfigs; - - @Value("#{'${queue.type:null}' == 'kafka'}") - private boolean isKafkaSupported; - @PostConstruct public void init() { log.debug("EdgeEventSourcingListener initiated"); @@ -127,11 +107,7 @@ public class EdgeEventSourcingListener { return; } try { - if (EntityType.TENANT.equals(entityType)) { - return; - } - if (EntityType.EDGE.equals(entityType)) { - handleEdgeEntityDeletion((EdgeId) event.getEntityId(), tenantId); + if (EntityType.TENANT.equals(entityType) || EntityType.EDGE.equals(entityType)) { return; } log.trace("[{}] DeleteEntityEvent called: {}", tenantId, event); @@ -145,14 +121,6 @@ public class EdgeEventSourcingListener { } } - private void handleEdgeEntityDeletion(EdgeId edgeId, TenantId tenantId) { - if (isKafkaSupported) { - String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); - TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); - kafkaAdmin.deleteTopic(topic); - } - } - private EdgeEventActionType getEdgeEventActionTypeForEntityEvent(Object entity) { if (entity instanceof AlarmComment) { return EdgeEventActionType.DELETED_COMMENT; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java index 15ee44a66e..2f06b4d6e4 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java @@ -539,7 +539,8 @@ public abstract class AbstractEdgeGrpcSession highPriorityEvents = new ArrayList<>(); EdgeEvent event; @@ -553,7 +554,8 @@ public abstract class AbstractEdgeGrpcSession processEdgeEvents() throws Exception { + @Override + public ListenableFuture processEdgeEvents() throws Exception { SettableFuture result = SettableFuture.create(); log.trace("[{}][{}] starting processing edge events", tenantId, sessionId); if (isConnected() && isSyncCompleted()) { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java index a82ba3efb2..dbb2166d69 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java @@ -57,9 +57,6 @@ import org.thingsboard.server.common.msg.edge.ToEdgeSyncRequest; import org.thingsboard.server.gen.edge.v1.EdgeRpcServiceGrpc; import org.thingsboard.server.gen.edge.v1.RequestMsg; import org.thingsboard.server.gen.edge.v1.ResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -95,7 +92,6 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i private final ConcurrentMap> sessionEdgeEventChecks = new ConcurrentHashMap<>(); private final ConcurrentMap> localSyncEdgeRequests = new ConcurrentHashMap<>(); private final ConcurrentMap edgeEventsProcessed = new ConcurrentHashMap<>(); - private final ConcurrentMap kafkaConsumerInit = new ConcurrentHashMap<>(); @Value("${edges.rpc.port}") private int rpcPort; @@ -220,7 +216,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i private AbstractEdgeGrpcSession createEdgeGrpcSession(StreamObserver outputStream) { return isKafkaSupported - ? new KafkaEdgeGrpcSession(ctx, outputStream, this::onEdgeConnect, this::onEdgeDisconnect, + ? new KafkaEdgeGrpcSession(ctx, tbCoreQueueFactory, outputStream, this::onEdgeConnect, this::onEdgeDisconnect, sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession) : new PostgresEdgeGrpcSession(ctx, outputStream, this::onEdgeConnect, this::onEdgeDisconnect, sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession); @@ -333,26 +329,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i pushRuleEngineMessage(tenantId, edge, lastConnectTs, TbMsgType.CONNECT_EVENT); cancelScheduleEdgeEventsCheck(edgeId); edgeEventsProcessed.putIfAbsent(edgeId, Boolean.FALSE); - if (edgeGrpcSession instanceof KafkaEdgeGrpcSession session) { - Boolean isChecked = edgeEventsProcessed.get(edgeId); - if (Boolean.FALSE.equals(isChecked)) { - scheduleEdgeEventsCheck(session); - } else { - initializeKafkaConsumer(session, tenantId, edgeId); - } - } - if (edgeGrpcSession instanceof PostgresEdgeGrpcSession) { - scheduleEdgeEventsCheck(edgeGrpcSession); - } - } - - private void initializeKafkaConsumer(KafkaEdgeGrpcSession kafkaEdgeGrpcSession, TenantId tenantId, EdgeId edgeId) { - TbQueueConsumer> consumer = tbCoreQueueFactory.createEdgeEventMsgConsumer(tenantId, edgeId); - if (!kafkaConsumerInit.getOrDefault(edgeId, Boolean.FALSE)) { - kafkaEdgeGrpcSession.initConsumer(() -> consumer, schedulerPoolSize); - kafkaEdgeGrpcSession.startConsumers(); - kafkaConsumerInit.put(edgeId, Boolean.TRUE); - } + scheduleEdgeEventsCheck(edgeGrpcSession); } private void startSyncProcess(TenantId tenantId, EdgeId edgeId, UUID requestId, String requestServiceId) { @@ -419,22 +396,18 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock()); newEventLock.lock(); try { - if (Boolean.TRUE.equals(sessionNewEvents.get(edgeId)) && Boolean.FALSE.equals(edgeEventsProcessed.get(edgeId))) { + if (Boolean.TRUE.equals(sessionNewEvents.get(edgeId))) { log.trace("[{}][{}] Set session new events flag to false", tenantId, edgeId.getId()); sessionNewEvents.put(edgeId, false); + processEdgeEventMigrationIfNeeded(session, edgeId); + session.processHighPriorityEvents(); Futures.addCallback(session.processEdgeEvents(), new FutureCallback<>() { @Override public void onSuccess(Boolean newEventsAdded) { if (Boolean.TRUE.equals(newEventsAdded)) { sessionNewEvents.put(edgeId, true); } - if (session instanceof KafkaEdgeGrpcSession kafkaEdgeGrpcSession && newEventsAdded != null) { - edgeEventsProcessed.put(edgeId, true); - initializeKafkaConsumer(kafkaEdgeGrpcSession, tenantId, edgeId); - cancelScheduleEdgeEventsCheck(edgeId); - } else { - scheduleEdgeEventsCheck(session); - } + scheduleEdgeEventsCheck(session); } @Override @@ -444,12 +417,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } }, ctx.getGrpcCallbackExecutorService()); } else { - if (Boolean.TRUE.equals(edgeEventsProcessed.get(edgeId)) && session instanceof KafkaEdgeGrpcSession kafkaEdgeGrpcSession) { - initializeKafkaConsumer(kafkaEdgeGrpcSession, tenantId, edgeId); - cancelScheduleEdgeEventsCheck(edgeId); - } else { - scheduleEdgeEventsCheck(session); - } + scheduleEdgeEventsCheck(session); } } finally { newEventLock.unlock(); @@ -466,6 +434,21 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } } + private void processEdgeEventMigrationIfNeeded(AbstractEdgeGrpcSession session, EdgeId edgeId) throws Exception { + boolean isMigrationProcessed = edgeEventsProcessed.getOrDefault(edgeId, Boolean.FALSE); + if (!isMigrationProcessed) { + Boolean migrated = session.migrateEdgeEvents(false).get(); + if (Boolean.TRUE.equals(migrated)) { + sessionNewEvents.put(edgeId, true); + scheduleEdgeEventsCheck(session); + } else if (Boolean.FALSE.equals(migrated)) { + edgeEventsProcessed.put(edgeId, true); + } else { + scheduleEdgeEventsCheck(session); + } + } + } + private void cancelScheduleEdgeEventsCheck(EdgeId edgeId) { log.trace("[{}] cancelling edge event check for edge", edgeId); if (sessionEdgeEventChecks.containsKey(edgeId)) { @@ -492,7 +475,6 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } if (isKafkaSupported) { ((KafkaEdgeGrpcSession) toRemove).stopConsumer(); - kafkaConsumerInit.remove(edgeId); } TenantId tenantId = toRemove.getEdge().getTenantId(); save(tenantId, edgeId, DefaultDeviceStateService.ACTIVITY_STATE, false); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index 6a72b40b80..3db3a72288 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.edge.rpc; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.edge.Edge; public interface EdgeGrpcSession { @@ -25,4 +26,10 @@ public interface EdgeGrpcSession { boolean isConnected(); + ListenableFuture migrateEdgeEvents(boolean isMigrationProcessed) throws Exception; + + ListenableFuture processEdgeEvents() throws Exception; + + void processHighPriorityEvents(); + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java index e16c89c23e..4fabfd5945 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.service.edge.rpc; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import io.grpc.stub.StreamObserver; import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; @@ -29,6 +31,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificat import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.service.edge.EdgeContextComponent; import java.util.ArrayList; @@ -37,86 +40,50 @@ import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; -import java.util.function.Supplier; @Slf4j public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession { - private ExecutorService consumerExecutor; - private ScheduledExecutorService highPriorityExecutorService; + private final TbQueueConsumer> edgeEventsConsumer; + + private volatile boolean isHighPriorityProcessing; + private volatile boolean isConsumerInit; private QueueConsumerManager> consumer; - public KafkaEdgeGrpcSession(EdgeContextComponent ctx, StreamObserver outputStream, + private ExecutorService consumerExecutor; + + public KafkaEdgeGrpcSession(EdgeContextComponent ctx, TbCoreQueueFactory tbCoreQueueFactory, StreamObserver outputStream, BiConsumer sessionOpenListener, - BiConsumer sessionCloseListener, - ScheduledExecutorService sendDownlinkExecutorService, + BiConsumer sessionCloseListener, ScheduledExecutorService sendDownlinkExecutorService, int maxInboundMessageSize, int maxHighPriorityQueueSizePerSession) { super(ctx, outputStream, sessionOpenListener, sessionCloseListener, sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession); - } - - protected void initConsumer(Supplier>> edgeEventsConsumer, long schedulerPoolSize) { - this.consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("edge-event-consumer")); - this.highPriorityExecutorService = Executors.newScheduledThreadPool((int) schedulerPoolSize, ThingsBoardThreadFactory.forName("edge-event-high-priority-scheduler")); - this.consumer = QueueConsumerManager.>builder() - .name("TB Edge events") - .msgPackProcessor(this::processMsgs) - .pollInterval(ctx.getEdgeEventStorageSettings().getNoRecordsSleepInterval()) - .consumerCreator(edgeEventsConsumer) - .consumerExecutor(consumerExecutor) - .threadPrefix("edge-events") - .build(); - scheduleCheckForHighPriorityEvent(); + edgeEventsConsumer = tbCoreQueueFactory.createEdgeEventMsgConsumer(tenantId, edge.getId()); } private void processMsgs(List> msgs, TbQueueConsumer> consumer) { log.trace("[{}][{}] starting processing edge events", tenantId, sessionId); - if (isConnected() && isSyncCompleted()) { - - if (!highPriorityQueue.isEmpty()) { - processHighPriorityEvents(); - } else { - List edgeEvents = new ArrayList<>(); - for (TbProtoQueueMsg msg : msgs) { - EdgeEvent edgeEvent = ProtoUtils.fromProto(msg.getValue().getEdgeEventMsg()); - edgeEvents.add(edgeEvent); - } - List downlinkMsgsPack = convertToDownlinkMsgsPack(edgeEvents); - try { - boolean isInterrupted = sendDownlinkMsgsPack(downlinkMsgsPack).get(); - if (isInterrupted) { - log.debug("[{}][{}][{}] Send downlink messages task was interrupted", tenantId, edge.getId(), sessionId); - } else { - consumer.commit(); - } - } catch (Exception e) { - log.error("[{}] Failed to process all downlink messages", sessionId, e); - } + if (isConnected() && isSyncCompleted() && !isHighPriorityProcessing) { + List edgeEvents = new ArrayList<>(); + for (TbProtoQueueMsg msg : msgs) { + EdgeEvent edgeEvent = ProtoUtils.fromProto(msg.getValue().getEdgeEventMsg()); + edgeEvents.add(edgeEvent); } - } else { - log.trace("[{}][{}] edge is not connected or sync is not completed. Skipping iteration", tenantId, sessionId); - } - } - - private void scheduleCheckForHighPriorityEvent() { - highPriorityExecutorService.scheduleAtFixedRate(() -> { + List downlinkMsgsPack = convertToDownlinkMsgsPack(edgeEvents); try { - if (isConnected() && isSyncCompleted()) { - if (!highPriorityQueue.isEmpty()) { - processHighPriorityEvents(); - } + boolean isInterrupted = sendDownlinkMsgsPack(downlinkMsgsPack).get(); + if (isInterrupted) { + log.debug("[{}][{}][{}] Send downlink messages task was interrupted", tenantId, edge.getId(), sessionId); + } else { + consumer.commit(); } } catch (Exception e) { - log.error("Error in processing high priority events", e); + log.error("[{}] Failed to process all downlink messages", sessionId, e); } - }, 0, ctx.getEdgeEventStorageSettings().getNoRecordsSleepInterval() * 3, TimeUnit.MILLISECONDS); - } - - public void startConsumers() { - consumer.subscribe(); - consumer.launch(); + } else { + log.trace("[{}][{}] edge is not connected or sync is not completed. Skipping iteration", tenantId, sessionId); + } } @PreDestroy @@ -130,4 +97,37 @@ public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession migrateEdgeEvents(boolean isMigrationProcessed) throws Exception { + if (isMigrationProcessed) { + return Futures.immediateFuture(Boolean.FALSE); + } + return super.processEdgeEvents(); + } + + @Override + public ListenableFuture processEdgeEvents() { + if (!isConsumerInit) { + this.consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("edge-event-consumer")); + this.consumer = QueueConsumerManager.>builder() + .name("TB Edge events") + .msgPackProcessor(this::processMsgs) + .pollInterval(ctx.getEdgeEventStorageSettings().getNoRecordsSleepInterval()) + .consumerCreator(() -> edgeEventsConsumer) + .consumerExecutor(consumerExecutor) + .threadPrefix("edge-events") + .build(); + isConsumerInit = true; + consumer.subscribe(); + consumer.launch(); + } + return Futures.immediateFuture(Boolean.FALSE); + } + + @Override + public void processHighPriorityEvents() { + isHighPriorityProcessing = true; + super.processHighPriorityEvents(); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java index 4ef55e5961..d8f83c393a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.service.edge.rpc; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import io.grpc.stub.StreamObserver; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.edge.Edge; @@ -37,4 +39,9 @@ public class PostgresEdgeGrpcSession extends AbstractEdgeGrpcSession migrateEdgeEvents(boolean isMigrationProcessed) { + return Futures.immediateFuture(Boolean.FALSE); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 863be23e42..a613a2b5a1 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.entitiy; import com.fasterxml.jackson.core.type.TypeReference; import jakarta.annotation.PostConstruct; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.transaction.event.TransactionalEventListener; @@ -35,6 +34,7 @@ import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; @@ -54,17 +54,34 @@ import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import java.util.Optional; import java.util.Set; -@Component -@RequiredArgsConstructor @Slf4j +@Component public class EntityStateSourcingListener { + private final TopicService topicService; private final TbClusterService tbClusterService; private final TenantService tenantService; + private final Optional kafkaSettings; + private final Optional kafkaTopicConfigs; + + public EntityStateSourcingListener(TopicService topicService, TbClusterService tbClusterService, TenantService tenantService, + Optional kafkaSettings, Optional kafkaTopicConfigs) { + this.topicService = topicService; + this.tbClusterService = tbClusterService; + this.tenantService = tenantService; + this.kafkaSettings = kafkaSettings; + this.kafkaTopicConfigs = kafkaTopicConfigs; + } + @PostConstruct public void init() { log.debug("EntityStateSourcingListener initiated"); @@ -137,7 +154,7 @@ public class EntityStateSourcingListener { log.debug("[{}][{}][{}] Handling entity deletion event: {}", tenantId, entityType, entityId, event); switch (entityType) { - case ASSET, ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE -> { + case ASSET, ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } case NOTIFICATION_REQUEST -> { @@ -177,6 +194,10 @@ public class EntityStateSourcingListener { TbResourceInfo tbResource = (TbResourceInfo) event.getEntity(); tbClusterService.onResourceDeleted(tbResource, null); } + case EDGE -> { + onEdgeDelete(tenantId, (EdgeId) event.getEntityId()); + tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); + } default -> {} } } @@ -247,6 +268,14 @@ public class EntityStateSourcingListener { } } + private void onEdgeDelete(TenantId tenantId, EdgeId edgeId) { + if (kafkaSettings.isPresent() && kafkaTopicConfigs.isPresent()) { + String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); + TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings.get(), kafkaTopicConfigs.get().getEdgeEventConfigs()); + kafkaAdmin.deleteTopic(topic); + } + } + private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { String data = JacksonUtil.toString(JacksonUtil.valueToTree(assignedDevice)); if (data != null) { From aae9ac9edae57aa397fd97fcf00357330b6c3007 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Tue, 12 Nov 2024 17:45:15 +0200 Subject: [PATCH 31/71] added fix for SEQUENTIAL_ON_RESPONSE_FROM_DEVICE rpc submit strategy --- .../server/actors/device/DeviceActorMessageProcessor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 3a000bfe4d..04452d3fb6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -257,8 +257,7 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso private boolean isSendNewRpcAvailable() { return switch (rpcSubmitStrategy) { case SEQUENTIAL_ON_ACK_FROM_DEVICE -> toDeviceRpcPendingMap.values().stream().filter(md -> !md.isDelivered()).findAny().isEmpty(); - case SEQUENTIAL_ON_RESPONSE_FROM_DEVICE -> - toDeviceRpcPendingMap.values().stream().filter(ToDeviceRpcRequestMetadata::isDelivered).findAny().isEmpty(); + case SEQUENTIAL_ON_RESPONSE_FROM_DEVICE -> toDeviceRpcPendingMap.isEmpty(); default -> true; }; } From 732344c8d7d273b2cc9d8040dc62284608416a69 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 15 Nov 2024 14:44:44 +0200 Subject: [PATCH 32/71] Improvement after review --- .../edge/rpc/AbstractEdgeGrpcSession.java | 13 ++-- .../service/edge/rpc/EdgeGrpcService.java | 36 ++++++----- .../service/edge/rpc/EdgeGrpcSession.java | 4 +- .../edge/rpc/KafkaEdgeEventService.java | 41 ++----------- .../edge/rpc/KafkaEdgeGrpcSession.java | 36 +++++------ .../edge/rpc/PostgresEdgeGrpcSession.java | 9 ++- .../edge/rpc/processor/BaseEdgeProcessor.java | 16 ++--- .../processor/alarm/AlarmEdgeProcessor.java | 41 +++---------- .../relation/BaseRelationProcessor.java | 3 +- .../entitiy/EntityStateSourcingListener.java | 11 +--- .../queue/DefaultTbCoreConsumerService.java | 2 +- .../service/ttl/EdgeEventsCleanUpService.java | 1 + .../ttl/KafkaEdgeTopicsCleanUpService.java | 36 +++++------ .../server/dao/edge/EdgeEventService.java | 4 -- .../server/dao/edge/BaseEdgeEventService.java | 59 +++++++++++++++++++ .../server/dao/edge/EdgeEventDao.java | 2 - .../dao/edge/PostgresEdgeEventService.java | 32 +--------- .../dao/sql/edge/JpaBaseEdgeEventDao.java | 30 ---------- .../dao/service/EdgeEventServiceTest.java | 8 ++- 19 files changed, 154 insertions(+), 230 deletions(-) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/edge/BaseEdgeEventService.java diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java index 2f06b4d6e4..9fd94b4988 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java @@ -105,7 +105,7 @@ import java.util.function.BiConsumer; @Slf4j @Data -public abstract class AbstractEdgeGrpcSession> implements EdgeGrpcSession, Closeable { +public abstract class AbstractEdgeGrpcSession implements EdgeGrpcSession, Closeable { private static final String QUEUE_START_TS_ATTR_KEY = "queueStartTs"; private static final String QUEUE_START_SEQ_ID_ATTR_KEY = "queueStartSeqId"; @@ -116,7 +116,7 @@ public abstract class AbstractEdgeGrpcSession highPriorityQueue = new ConcurrentLinkedQueue<>(); protected UUID sessionId; - private BiConsumer sessionOpenListener; + private BiConsumer sessionOpenListener; private BiConsumer sessionCloseListener; private final EdgeSessionState sessionState = new EdgeSessionState(); @@ -146,7 +146,7 @@ public abstract class AbstractEdgeGrpcSession outputStream, - BiConsumer sessionOpenListener, + BiConsumer sessionOpenListener, BiConsumer sessionCloseListener, ScheduledExecutorService sendDownlinkExecutorService, int maxInboundMessageSize, int maxHighPriorityQueueSizePerSession) { @@ -299,11 +299,6 @@ public abstract class AbstractEdgeGrpcSession downlinkMsgsPack = convertToDownlinkMsgsPack(pageData.getData()); - for (DownlinkMsg downlinkMsg : downlinkMsgsPack) { - if (downlinkMsg.getEntityDataCount() > 0) { - System.out.println("downlink = " + downlinkMsg); - } - } Futures.addCallback(sendDownlinkMsgsPack(downlinkMsgsPack), new FutureCallback<>() { @Override public void onSuccess(@Nullable Boolean isInterrupted) { @@ -351,7 +346,7 @@ public abstract class AbstractEdgeGrpcSession> sessions = new ConcurrentHashMap<>(); + private final ConcurrentMap sessions = new ConcurrentHashMap<>(); private final ConcurrentMap sessionNewEventsLocks = new ConcurrentHashMap<>(); private final Map sessionNewEvents = new HashMap<>(); private final ConcurrentMap> sessionEdgeEventChecks = new ConcurrentHashMap<>(); @@ -210,11 +210,11 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i @Override public StreamObserver handleMsgs(StreamObserver outputStream) { - AbstractEdgeGrpcSession session = createEdgeGrpcSession(outputStream); + AbstractEdgeGrpcSession session = createEdgeGrpcSession(outputStream); return session.getInputStream(); } - private AbstractEdgeGrpcSession createEdgeGrpcSession(StreamObserver outputStream) { + private AbstractEdgeGrpcSession createEdgeGrpcSession(StreamObserver outputStream) { return isKafkaSupported ? new KafkaEdgeGrpcSession(ctx, tbCoreQueueFactory, outputStream, this::onEdgeConnect, this::onEdgeDisconnect, sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession) @@ -250,7 +250,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i @Override public void updateEdge(TenantId tenantId, Edge edge) { - AbstractEdgeGrpcSession session = sessions.get(edge.getId()); + AbstractEdgeGrpcSession session = sessions.get(edge.getId()); if (session != null && session.isConnected()) { log.debug("[{}] Updating configuration for edge [{}] [{}]", tenantId, edge.getName(), edge.getId()); session.onConfigurationUpdate(edge); @@ -261,7 +261,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i @Override public void deleteEdge(TenantId tenantId, EdgeId edgeId) { - AbstractEdgeGrpcSession session = sessions.get(edgeId); + AbstractEdgeGrpcSession session = sessions.get(edgeId); if (session != null && session.isConnected()) { log.info("[{}] Closing and removing session for edge [{}]", tenantId, edgeId); session.close(); @@ -278,7 +278,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } private void onEdgeEventUpdate(TenantId tenantId, EdgeId edgeId) { - AbstractEdgeGrpcSession session = sessions.get(edgeId); + AbstractEdgeGrpcSession session = sessions.get(edgeId); if (session != null && session.isConnected()) { log.trace("[{}] onEdgeEventUpdate [{}]", tenantId, edgeId.getId()); updateSessionEventsFlag(tenantId, edgeId); @@ -289,7 +289,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i TenantId tenantId = msg.getTenantId(); EdgeEvent edgeEvent = msg.getEdgeEvent(); EdgeId edgeId = edgeEvent.getEdgeId(); - AbstractEdgeGrpcSession session = sessions.get(edgeId); + AbstractEdgeGrpcSession session = sessions.get(edgeId); if (session != null && session.isConnected()) { log.trace("[{}] onEdgeEvent [{}]", tenantId, edgeId); session.addEventToHighPriorityQueue(edgeEvent); @@ -310,7 +310,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } } - private void onEdgeConnect(EdgeId edgeId, AbstractEdgeGrpcSession edgeGrpcSession) { + private void onEdgeConnect(EdgeId edgeId, AbstractEdgeGrpcSession edgeGrpcSession) { Edge edge = edgeGrpcSession.getEdge(); TenantId tenantId = edge.getTenantId(); log.info("[{}][{}] edge [{}] connected successfully.", tenantId, edgeGrpcSession.getSessionId(), edgeId); @@ -333,7 +333,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } private void startSyncProcess(TenantId tenantId, EdgeId edgeId, UUID requestId, String requestServiceId) { - AbstractEdgeGrpcSession session = sessions.get(edgeId); + AbstractEdgeGrpcSession session = sessions.get(edgeId); if (session != null) { if (!session.isSyncCompleted()) { clusterService.pushEdgeSyncResponseToCore(new FromEdgeSyncResponse(requestId, tenantId, edgeId, false, "Sync process is active at the moment"), requestServiceId); @@ -353,7 +353,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i ToEdgeSyncRequest request = new ToEdgeSyncRequest(UUID.randomUUID(), tenantId, edgeId, serviceInfoProvider.getServiceId()); UUID requestId = request.getId(); - AbstractEdgeGrpcSession session = sessions.get(request.getEdgeId()); + AbstractEdgeGrpcSession session = sessions.get(request.getEdgeId()); if (session != null && !session.isSyncCompleted()) { responseConsumer.accept(new FromEdgeSyncResponse(requestId, request.getTenantId(), request.getEdgeId(), false, "Sync process is active at the moment")); } else { @@ -387,7 +387,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } } - private void scheduleEdgeEventsCheck(AbstractEdgeGrpcSession session) { + private void scheduleEdgeEventsCheck(AbstractEdgeGrpcSession session) { EdgeId edgeId = session.getEdge().getId(); TenantId tenantId = session.getEdge().getTenantId(); if (sessions.containsKey(edgeId)) { @@ -434,14 +434,14 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } } - private void processEdgeEventMigrationIfNeeded(AbstractEdgeGrpcSession session, EdgeId edgeId) throws Exception { + private void processEdgeEventMigrationIfNeeded(AbstractEdgeGrpcSession session, EdgeId edgeId) throws Exception { boolean isMigrationProcessed = edgeEventsProcessed.getOrDefault(edgeId, Boolean.FALSE); if (!isMigrationProcessed) { - Boolean migrated = session.migrateEdgeEvents(false).get(); - if (Boolean.TRUE.equals(migrated)) { + Boolean eventsExist = session.migrateEdgeEvents().get(); + if (Boolean.TRUE.equals(eventsExist)) { sessionNewEvents.put(edgeId, true); scheduleEdgeEventsCheck(session); - } else if (Boolean.FALSE.equals(migrated)) { + } else if (Boolean.FALSE.equals(eventsExist)) { edgeEventsProcessed.put(edgeId, true); } else { scheduleEdgeEventsCheck(session); @@ -463,7 +463,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i private void onEdgeDisconnect(Edge edge, UUID sessionId) { EdgeId edgeId = edge.getId(); log.info("[{}][{}] edge disconnected!", edgeId, sessionId); - AbstractEdgeGrpcSession toRemove = sessions.get(edgeId); + AbstractEdgeGrpcSession toRemove = sessions.get(edgeId); if (toRemove.getSessionId().equals(sessionId)) { toRemove = sessions.remove(edgeId); final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock()); @@ -473,9 +473,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } finally { newEventLock.unlock(); } - if (isKafkaSupported) { - ((KafkaEdgeGrpcSession) toRemove).stopConsumer(); - } + toRemove.destroy(); TenantId tenantId = toRemove.getEdge().getTenantId(); save(tenantId, edgeId, DefaultDeviceStateService.ACTIVITY_STATE, false); long lastDisconnectTs = System.currentTimeMillis(); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index 3db3a72288..880712d6b2 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -26,7 +26,9 @@ public interface EdgeGrpcSession { boolean isConnected(); - ListenableFuture migrateEdgeEvents(boolean isMigrationProcessed) throws Exception; + void destroy(); + + ListenableFuture migrateEdgeEvents() throws Exception; ListenableFuture processEdgeEvents() throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java index 1126214e60..9b93238400 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java @@ -22,20 +22,10 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; -import org.thingsboard.server.cache.limits.RateLimitService; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.edge.EdgeEvent; -import org.thingsboard.server.common.data.id.EdgeId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.limit.LimitedApi; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.util.ProtoUtils; -import org.thingsboard.server.dao.edge.EdgeEventDao; -import org.thingsboard.server.dao.edge.EdgeEventService; -import org.thingsboard.server.dao.service.DataValidator; +import org.thingsboard.server.dao.edge.BaseEdgeEventService; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TopicService; @@ -47,25 +37,16 @@ import java.util.UUID; @Service @RequiredArgsConstructor @ConditionalOnExpression("'${queue.type:null}'=='kafka'") -public class KafkaEdgeEventService implements EdgeEventService { +public class KafkaEdgeEventService extends BaseEdgeEventService { - private final RateLimitService rateLimitService; - private final DataValidator edgeEventValidator; - @Lazy - private final TbQueueProducerProvider producerProvider; @Lazy private final TopicService topicService; - private final EdgeEventDao edgeEventDao; + @Lazy + private final TbQueueProducerProvider producerProvider; @Override public ListenableFuture saveAsync(EdgeEvent edgeEvent) { - if (!rateLimitService.checkRateLimit(LimitedApi.EDGE_EVENTS, edgeEvent.getTenantId())) { - throw new TbRateLimitsException(EntityType.TENANT); - } - if (!rateLimitService.checkRateLimit(LimitedApi.EDGE_EVENTS_PER_EDGE, edgeEvent.getTenantId(), edgeEvent.getEdgeId())) { - throw new TbRateLimitsException(EntityType.EDGE); - } - edgeEventValidator.validate(edgeEvent, EdgeEvent::getTenantId); + validateEdgeEvent(edgeEvent); TopicPartitionInfo tpi = topicService.getEdgeEventNotificationsTopic(edgeEvent.getTenantId(), edgeEvent.getEdgeId()); ToEdgeEventNotificationMsg msg = ToEdgeEventNotificationMsg.newBuilder().setEdgeEventMsg(ProtoUtils.toProto(edgeEvent)).build(); @@ -74,16 +55,4 @@ public class KafkaEdgeEventService implements EdgeEventService { return Futures.immediateFuture(null); } - @Override - public PageData findEdgeEvents(TenantId tenantId, EdgeId edgeId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink) { - // To support fetching edge events on connect from postgres if there are any: - return edgeEventDao.findEdgeEvents(tenantId.getId(), edgeId, seqIdStart, seqIdEnd, pageLink); - } - - @Override - public void cleanupEvents(long ttl) { - // To delete deprecated events by ttl - edgeEventDao.cleanupEvents(ttl); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java index 4fabfd5945..15a87ec3f1 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.edge.rpc; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import io.grpc.stub.StreamObserver; -import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.edge.Edge; @@ -43,19 +42,18 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.function.BiConsumer; @Slf4j -public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession { +public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession { private final TbQueueConsumer> edgeEventsConsumer; private volatile boolean isHighPriorityProcessing; - private volatile boolean isConsumerInit; private QueueConsumerManager> consumer; private ExecutorService consumerExecutor; public KafkaEdgeGrpcSession(EdgeContextComponent ctx, TbCoreQueueFactory tbCoreQueueFactory, StreamObserver outputStream, - BiConsumer sessionOpenListener, + BiConsumer sessionOpenListener, BiConsumer sessionCloseListener, ScheduledExecutorService sendDownlinkExecutorService, int maxInboundMessageSize, int maxHighPriorityQueueSizePerSession) { super(ctx, outputStream, sessionOpenListener, sessionCloseListener, sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession); @@ -82,32 +80,23 @@ public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession migrateEdgeEvents(boolean isMigrationProcessed) throws Exception { - if (isMigrationProcessed) { - return Futures.immediateFuture(Boolean.FALSE); - } + public ListenableFuture migrateEdgeEvents() throws Exception { return super.processEdgeEvents(); } @Override public ListenableFuture processEdgeEvents() { - if (!isConsumerInit) { + if (consumer == null) { this.consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("edge-event-consumer")); this.consumer = QueueConsumerManager.>builder() .name("TB Edge events") @@ -117,7 +106,6 @@ public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession { +public class PostgresEdgeGrpcSession extends AbstractEdgeGrpcSession { PostgresEdgeGrpcSession(EdgeContextComponent ctx, StreamObserver outputStream, - BiConsumer sessionOpenListener, + BiConsumer sessionOpenListener, BiConsumer sessionCloseListener, ScheduledExecutorService sendDownlinkExecutorService, int maxInboundMessageSize, int maxHighPriorityQueueSizePerSession) { super(ctx, outputStream, sessionOpenListener, sessionCloseListener, sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession); @@ -40,7 +40,10 @@ public class PostgresEdgeGrpcSession extends AbstractEdgeGrpcSession migrateEdgeEvents(boolean isMigrationProcessed) { + public void destroy() {} + + @Override + public ListenableFuture migrateEdgeEvents() { return Futures.immediateFuture(Boolean.FALSE); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index e8e2ca7c3a..e448fbd937 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -51,6 +51,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.edge.EdgeSynchronizationManager; +import org.thingsboard.server.dao.entity.EntityDaoRegistry; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueCallback; @@ -77,6 +78,9 @@ public abstract class BaseEdgeProcessor { @Autowired protected EdgeContextComponent edgeCtx; + @Autowired + protected EntityDaoRegistry entityDaoRegistry; + @Autowired protected EdgeSynchronizationManager edgeSynchronizationManager; @@ -315,17 +319,7 @@ public abstract class BaseEdgeProcessor { } protected boolean isEntityExists(TenantId tenantId, EntityId entityId) { - return switch (entityId.getEntityType()) { - case TENANT -> edgeCtx.getTenantService().findTenantById(tenantId) != null; - case DEVICE -> edgeCtx.getDeviceService().findDeviceById(tenantId, new DeviceId(entityId.getId())) != null; - case ASSET -> edgeCtx.getAssetService().findAssetById(tenantId, new AssetId(entityId.getId())) != null; - case ENTITY_VIEW -> edgeCtx.getEntityViewService().findEntityViewById(tenantId, new EntityViewId(entityId.getId())) != null; - case CUSTOMER -> edgeCtx.getCustomerService().findCustomerById(tenantId, new CustomerId(entityId.getId())) != null; - case USER -> edgeCtx.getUserService().findUserById(tenantId, new UserId(entityId.getId())) != null; - case DASHBOARD -> edgeCtx.getDashboardService().findDashboardById(tenantId, new DashboardId(entityId.getId())) != null; - case EDGE -> edgeCtx.getEdgeService().findEdgeById(tenantId, new EdgeId(entityId.getId())) != null; - default -> false; - }; + return entityDaoRegistry.getDao(entityId.getEntityType()).existsById(tenantId, entityId.getId()); } protected void createRelationFromEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId) { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java index 650aafb8cc..f0a8012660 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/alarm/AlarmEdgeProcessor.java @@ -21,23 +21,18 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EdgeUtils; -import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmComment; -import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AlarmId; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterableByTenantIdEntityId; +import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; @@ -59,6 +54,9 @@ public abstract class AlarmEdgeProcessor extends BaseAlarmProcessor implements A @Autowired private AlarmMsgConstructorFactory alarmMsgConstructorFactory; + @Autowired + private EntityService entityService; + @Override public ListenableFuture processAlarmMsgFromEdge(TenantId tenantId, EdgeId edgeId, AlarmUpdateMsg alarmUpdateMsg) { log.trace("[{}] processAlarmMsgFromEdge [{}]", tenantId, alarmUpdateMsg); @@ -178,42 +176,19 @@ public abstract class AlarmEdgeProcessor extends BaseAlarmProcessor implements A case ADDED, UPDATED, ALARM_ACK, ALARM_CLEAR -> { Alarm alarm = edgeCtx.getAlarmService().findAlarmById(tenantId, alarmId); if (alarm != null) { - return msgConstructor.constructAlarmUpdatedMsg(msgType, alarm, findOriginatorEntityName(tenantId, alarm)); + return msgConstructor.constructAlarmUpdatedMsg(msgType, alarm, + entityService.fetchEntityName(tenantId, alarm.getOriginator()).orElse(null)); } } case ALARM_DELETE, DELETED -> { Alarm deletedAlarm = JacksonUtil.convertValue(body, Alarm.class); if (deletedAlarm != null) { - return msgConstructor.constructAlarmUpdatedMsg(msgType, deletedAlarm, findOriginatorEntityName(tenantId, deletedAlarm)); + return msgConstructor.constructAlarmUpdatedMsg(msgType, deletedAlarm, + entityService.fetchEntityName(tenantId, deletedAlarm.getOriginator()).orElse(null)); } } } return null; } - private String findOriginatorEntityName(TenantId tenantId, Alarm alarm) { - String entityName = null; - switch (alarm.getOriginator().getEntityType()) { - case DEVICE -> { - Device deviceById = edgeCtx.getDeviceService().findDeviceById(tenantId, new DeviceId(alarm.getOriginator().getId())); - if (deviceById != null) { - entityName = deviceById.getName(); - } - } - case ASSET -> { - Asset assetById = edgeCtx.getAssetService().findAssetById(tenantId, new AssetId(alarm.getOriginator().getId())); - if (assetById != null) { - entityName = assetById.getName(); - } - } - case ENTITY_VIEW -> { - EntityView entityViewById = edgeCtx.getEntityViewService().findEntityViewById(tenantId, new EntityViewId(alarm.getOriginator().getId())); - if (entityViewById != null) { - entityName = entityViewById.getName(); - } - } - } - return entityName; - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/BaseRelationProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/BaseRelationProcessor.java index 9ba863f62a..0c6da73a25 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/BaseRelationProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/BaseRelationProcessor.java @@ -36,8 +36,7 @@ public abstract class BaseRelationProcessor extends BaseEdgeProcessor { switch (relationUpdateMsg.getMsgType()) { case ENTITY_CREATED_RPC_MESSAGE: case ENTITY_UPDATED_RPC_MESSAGE: - if (isEntityExists(tenantId, entityRelation.getTo()) - && isEntityExists(tenantId, entityRelation.getFrom())) { + if (isEntityExists(tenantId, entityRelation.getTo()) && isEntityExists(tenantId, entityRelation.getFrom())) { edgeCtx.getRelationService().saveRelation(tenantId, entityRelation); } else { log.warn("[{}] Skipping relating update msg because from/to entity doesn't exists on edge, {}", tenantId, relationUpdateMsg); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index a613a2b5a1..38cebbb509 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.entitiy; import com.fasterxml.jackson.core.type.TypeReference; import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.transaction.event.TransactionalEventListener; @@ -64,6 +65,7 @@ import java.util.Set; @Slf4j @Component +@RequiredArgsConstructor public class EntityStateSourcingListener { private final TopicService topicService; @@ -73,15 +75,6 @@ public class EntityStateSourcingListener { private final Optional kafkaSettings; private final Optional kafkaTopicConfigs; - public EntityStateSourcingListener(TopicService topicService, TbClusterService tbClusterService, TenantService tenantService, - Optional kafkaSettings, Optional kafkaTopicConfigs) { - this.topicService = topicService; - this.tbClusterService = tbClusterService; - this.tenantService = tenantService; - this.kafkaSettings = kafkaSettings; - this.kafkaTopicConfigs = kafkaTopicConfigs; - } - @PostConstruct public void init() { log.debug("EntityStateSourcingListener initiated"); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 8e4af4fa28..2a64dd3388 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -236,7 +236,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService edgeIds = new PageDataIterable<>(link -> edgeService.findEdgeIdsByTenantId(tenantId, link), 1024); long currentTimeMillis = System.currentTimeMillis(); + long ttlMillis = TimeUnit.SECONDS.toChronoUnit().getDuration().multipliedBy(ttlSeconds).toMillis(); for (EdgeId edgeId : edgeIds) { - Optional attributeOpt = attributesService.find(tenantId, edgeId, AttributeScope.SERVER_SCOPE, DefaultDeviceStateService.LAST_CONNECT_TIME).get(); - if (attributeOpt.isPresent()) { - Optional lastConnectTimeOpt = attributeOpt.get().getLongValue(); - if (lastConnectTimeOpt.isPresent() && isTopicExpired(lastConnectTimeOpt.get(), currentTimeMillis)) { - String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); - TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); - if (kafkaAdmin.isTopicEmpty(topic)) { - kafkaAdmin.deleteTopic(topic); - log.info("Removed outdated topic for tenant {} and edge with id {} older than {}", tenantId, edgeId, Date.from(Instant.ofEpochMilli(currentTimeMillis - ONE_MONTH_MILLIS))); - } - } - } + attributesService.find(tenantId, edgeId, AttributeScope.SERVER_SCOPE, DefaultDeviceStateService.LAST_CONNECT_TIME).get() + .flatMap(AttributeKvEntry::getLongValue) + .filter(lastConnectTime -> isTopicExpired(lastConnectTime, ttlMillis, currentTimeMillis)) + .ifPresent(lastConnectTime -> { + String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); + TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + if (kafkaAdmin.isTopicEmpty(topic)) { + kafkaAdmin.deleteTopic(topic); + log.info("Removed outdated topic for tenant {} and edge with id {} older than {}", + tenantId, edgeId, Date.from(Instant.ofEpochMilli(currentTimeMillis - ttlMillis))); + } + }); } } - private boolean isTopicExpired(long lastConnectTime, long currentTimeMillis) { - return lastConnectTime + ONE_MONTH_MILLIS < currentTimeMillis; + private boolean isTopicExpired(long lastConnectTime, long ttlMillis, long currentTimeMillis) { + return lastConnectTime + ttlMillis < currentTimeMillis; } } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeEventService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeEventService.java index 9a7fc854d1..b7735228eb 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeEventService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeEventService.java @@ -28,10 +28,6 @@ public interface EdgeEventService { PageData findEdgeEvents(TenantId tenantId, EdgeId edgeId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink); - /** - * Executes stored procedure to cleanup old edge events. - * @param ttl the ttl for edge events in seconds - */ void cleanupEvents(long ttl); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/BaseEdgeEventService.java b/dao/src/main/java/org/thingsboard/server/dao/edge/BaseEdgeEventService.java new file mode 100644 index 0000000000..b8736e913a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/BaseEdgeEventService.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.edge; + +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.cache.limits.RateLimitService; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.limit.LimitedApi; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.common.msg.tools.TbRateLimitsException; +import org.thingsboard.server.dao.service.DataValidator; + +public abstract class BaseEdgeEventService implements EdgeEventService { + + @Autowired + private EdgeEventDao edgeEventDao; + @Autowired + private RateLimitService rateLimitService; + @Autowired + private DataValidator edgeEventValidator; + + @Override + public PageData findEdgeEvents(TenantId tenantId, EdgeId edgeId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink) { + return edgeEventDao.findEdgeEvents(tenantId.getId(), edgeId, seqIdStart, seqIdEnd, pageLink); + } + + @Override + public void cleanupEvents(long ttl) { + edgeEventDao.cleanupEvents(ttl); + } + + protected void validateEdgeEvent(EdgeEvent edgeEvent) { + if (!rateLimitService.checkRateLimit(LimitedApi.EDGE_EVENTS, edgeEvent.getTenantId())) { + throw new TbRateLimitsException(EntityType.TENANT); + } + if (!rateLimitService.checkRateLimit(LimitedApi.EDGE_EVENTS_PER_EDGE, edgeEvent.getTenantId(), edgeEvent.getEdgeId())) { + throw new TbRateLimitsException(EntityType.EDGE); + } + edgeEventValidator.validate(edgeEvent, EdgeEvent::getTenantId); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeEventDao.java index 1f9562724d..572733d75b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeEventDao.java @@ -56,6 +56,4 @@ public interface EdgeEventDao extends Dao { */ void cleanupEvents(long ttl); - void migrateEdgeEvents(); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/PostgresEdgeEventService.java b/dao/src/main/java/org/thingsboard/server/dao/edge/PostgresEdgeEventService.java index 8d23c8ee5d..b021ac90df 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/PostgresEdgeEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/PostgresEdgeEventService.java @@ -27,17 +27,8 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.cache.limits.RateLimitService; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.edge.EdgeEvent; -import org.thingsboard.server.common.data.id.EdgeId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.limit.LimitedApi; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; -import org.thingsboard.server.dao.service.DataValidator; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -46,11 +37,9 @@ import java.util.concurrent.Executors; @Service @RequiredArgsConstructor @ConditionalOnExpression("'${queue.type:null}'!='kafka'") -public class PostgresEdgeEventService implements EdgeEventService { +public class PostgresEdgeEventService extends BaseEdgeEventService { private final EdgeEventDao edgeEventDao; - private final RateLimitService rateLimitService; - private final DataValidator edgeEventValidator; private final ApplicationEventPublisher eventPublisher; private ExecutorService edgeEventExecutor; @@ -69,14 +58,7 @@ public class PostgresEdgeEventService implements EdgeEventService { @Override public ListenableFuture saveAsync(EdgeEvent edgeEvent) { - if (!rateLimitService.checkRateLimit(LimitedApi.EDGE_EVENTS, edgeEvent.getTenantId())) { - throw new TbRateLimitsException(EntityType.TENANT); - } - if (!rateLimitService.checkRateLimit(LimitedApi.EDGE_EVENTS_PER_EDGE, edgeEvent.getTenantId(), edgeEvent.getEdgeId())) { - throw new TbRateLimitsException(EntityType.EDGE); - } - edgeEventValidator.validate(edgeEvent, EdgeEvent::getTenantId); - + validateEdgeEvent(edgeEvent); ListenableFuture saveFuture = edgeEventDao.saveAsync(edgeEvent); Futures.addCallback(saveFuture, new FutureCallback<>() { @@ -96,14 +78,4 @@ public class PostgresEdgeEventService implements EdgeEventService { return saveFuture; } - @Override - public PageData findEdgeEvents(TenantId tenantId, EdgeId edgeId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink) { - return edgeEventDao.findEdgeEvents(tenantId.getId(), edgeId, seqIdStart, seqIdEnd, pageLink); - } - - @Override - public void cleanupEvents(long ttl) { - edgeEventDao.cleanupEvents(ttl); - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java index d9d2282fca..065a057160 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java @@ -198,36 +198,6 @@ public class JpaBaseEdgeEventDao extends JpaPartitionedAbstractDao 0 ? System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(edgeEventsTtl) : 1629158400000L; - - long currentTime = System.currentTimeMillis(); - var partitionStepInMs = TimeUnit.HOURS.toMillis(partitionSizeInHours); - long numberOfPartitions = (currentTime - startTime) / partitionStepInMs; - - if (numberOfPartitions > 1000) { - String error = "Please adjust your edge event partitioning configuration. Configuration with partition size " + - "of " + partitionSizeInHours + " hours and corresponding TTL will use " + numberOfPartitions + " " + - "(> 1000) partitions which is not recommended!"; - log.error(error); - throw new RuntimeException(error); - } - - while (startTime < currentTime) { - var endTime = startTime + partitionStepInMs; - log.info("Migrating edge event for time period: {} - {}", startTime, endTime); - callMigrationFunction(startTime, endTime, partitionStepInMs); - startTime = endTime; - } - log.info("Event edge migration finished"); - jdbcTemplate.execute("DROP TABLE IF EXISTS old_edge_event"); - } - - private void callMigrationFunction(long startTime, long endTime, long partitionSIzeInMs) { - jdbcTemplate.update("CALL migrate_edge_event(?, ?, ?)", startTime, endTime, partitionSIzeInMs); - } - @Override public void createPartition(EdgeEventEntity entity) { partitioningRepository.createPartitionIfNotExists(TABLE_NAME, entity.getCreatedTime(), TimeUnit.HOURS.toMillis(partitionSizeInHours)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EdgeEventServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EdgeEventServiceTest.java index 2cc179fae2..be8cb5d749 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EdgeEventServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EdgeEventServiceTest.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.dao.edge.EdgeEventDao; import org.thingsboard.server.dao.edge.EdgeEventService; import java.io.IOException; @@ -48,6 +49,9 @@ public class EdgeEventServiceTest extends AbstractServiceTest { @Autowired EdgeEventService edgeEventService; + @Autowired + EdgeEventDao edgeEventDao; + long timeBeforeStartTime; long startTime; long eventTime; @@ -81,7 +85,7 @@ public class EdgeEventServiceTest extends AbstractServiceTest { Assert.assertEquals(saved.getAction(), edgeEvent.getAction()); Assert.assertEquals(saved.getBody(), edgeEvent.getBody()); - edgeEventService.cleanupEvents(1); + edgeEventDao.cleanupEvents(1); } protected EdgeEvent generateEdgeEvent(TenantId tenantId, EdgeId edgeId, EntityId entityId) throws IOException { @@ -129,7 +133,7 @@ public class EdgeEventServiceTest extends AbstractServiceTest { Assert.assertEquals(Uuids.startOf(eventTime), edgeEvents.getData().get(0).getUuidId()); Assert.assertFalse(edgeEvents.hasNext()); - edgeEventService.cleanupEvents(1); + edgeEventDao.cleanupEvents(1); } private ListenableFuture saveEdgeEventWithProvidedTime(long time, EdgeId edgeId, EntityId entityId, TenantId tenantId) throws Exception { From 608e646ca8e30b4f1ad45e40046a277a2ac622b3 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Tue, 19 Nov 2024 09:43:14 +0200 Subject: [PATCH 33/71] Fixed non returning SubAck for provisioning clients --- .../transport/mqtt/MqttTransportHandler.java | 23 +++++++++- .../msa/connectivity/MqttClientTest.java | 43 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index d7f64e58ee..6d5c0f5d50 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -313,6 +313,28 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement sendResponseForAdaptorErrorOrCloseContext(ctx, topicName, msgId); } break; + case SUBSCRIBE: + MqttSubscribeMessage mqttSubscribeMsg = (MqttSubscribeMessage) msg; + log.trace("[{}] Processing subscription [{}]!", sessionId, mqttSubscribeMsg.variableHeader().messageId()); + List grantedQoSList = new ArrayList<>(); + for (MqttTopicSubscription subscription : mqttSubscribeMsg.payload().topicSubscriptions()) { + String topic = subscription.topicFilter(); + MqttQoS reqQoS = subscription.qualityOfService(); + try { + if (MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC.equals(topic)) { + registerSubQoS(topic, grantedQoSList, reqQoS); + } else { + log.warn("[{}] Failed to subscribe to [{}][{}]", sessionId, topic, reqQoS); + grantedQoSList.add(ReturnCodeResolver.getSubscriptionReturnCode(deviceSessionCtx.getMqttVersion(), MqttReasonCodes.SubAck.TOPIC_FILTER_INVALID)); + break; + } + } catch (Exception e) { + log.warn("[{}] Failed to subscribe to [{}][{}]", sessionId, topic, reqQoS, e); + grantedQoSList.add(ReturnCodeResolver.getSubscriptionReturnCode(deviceSessionCtx.getMqttVersion(), MqttReasonCodes.SubAck.IMPLEMENTATION_SPECIFIC_ERROR)); + } + } + ctx.writeAndFlush(createSubAckMessage(mqttSubscribeMsg.variableHeader().messageId(), grantedQoSList)); + break; case PINGREQ: ctx.writeAndFlush(new MqttMessage(new MqttFixedHeader(PINGRESP, false, AT_MOST_ONCE, false, 0))); break; @@ -822,7 +844,6 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement case MqttTopics.GATEWAY_ATTRIBUTES_TOPIC: case MqttTopics.GATEWAY_RPC_TOPIC: case MqttTopics.GATEWAY_ATTRIBUTES_RESPONSE_TOPIC: - case MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC: case MqttTopics.DEVICE_FIRMWARE_RESPONSES_TOPIC: case MqttTopics.DEVICE_FIRMWARE_ERROR_TOPIC: case MqttTopics.DEVICE_SOFTWARE_RESPONSES_TOPIC: diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java index c298d4fea3..8c26be9b45 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java @@ -28,6 +28,7 @@ import io.netty.handler.codec.mqtt.MqttMessage; import io.netty.handler.codec.mqtt.MqttQoS; import io.netty.handler.codec.mqtt.MqttReasonCodeAndPropertiesVariableHeader; import io.netty.handler.codec.mqtt.MqttReasonCodes; +import io.netty.handler.codec.mqtt.MqttSubAckMessage; import io.netty.handler.codec.mqtt.MqttVersion; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -491,6 +492,48 @@ public class MqttClientTest extends AbstractContainerTest { updateDeviceProfileWithProvisioningStrategy(deviceProfile, DeviceProfileProvisionType.DISABLED); } + @Test + public void provisionRequestForCheckSubAckReceived() throws Exception { + + DeviceProfile deviceProfile = testRestClient.getDeviceProfileById(device.getDeviceProfileId()); + deviceProfile = updateDeviceProfileWithProvisioningStrategy(deviceProfile, DeviceProfileProvisionType.ALLOW_CREATE_NEW_DEVICES); + + MqttMessageListener listener = new MqttMessageListener(); + MqttClient mqttClient = getMqttClient("provision", listener, MqttVersion.MQTT_5); + final MqttReasonCodes.SubAck[] subAckResult = new MqttReasonCodes.SubAck[1]; + mqttClient.setCallback(new MqttClientCallback() { + @Override + public void connectionLost(Throwable cause) { + } + + @Override + public void onSuccessfulReconnect() { + } + + @Override + public void onSubAck(MqttSubAckMessage subAckMessage) { + subAckResult[0] = subAckMessage.payload().typedReasonCodes().get(0); + } + } + ); + + mqttClient.on("/provision/response", listener, MqttQoS.AT_LEAST_ONCE).get(3 * timeoutMultiplier, TimeUnit.SECONDS); + TimeUnit.SECONDS.sleep(2 * timeoutMultiplier); + assertThat(subAckResult[0]).isNotNull(); + assertThat(MqttReasonCodes.SubAck.GRANTED_QOS_1.equals(subAckResult[0])); + + subAckResult[0] = null; + mqttClient.on("v1/devices/me/attributes", listener, MqttQoS.AT_LEAST_ONCE).get(3 * timeoutMultiplier, TimeUnit.SECONDS); + TimeUnit.SECONDS.sleep(2 * timeoutMultiplier); + assertThat(subAckResult[0]).isNotNull(); + assertThat(MqttReasonCodes.SubAck.TOPIC_FILTER_INVALID.equals(subAckResult[0])); + + testRestClient.deleteDeviceIfExists(device.getId()); + updateDeviceProfileWithProvisioningStrategy(deviceProfile, DeviceProfileProvisionType.DISABLED); + } + + + @Test public void provisionRequestForDeviceWithDisabledProvisioningStrategy() throws Exception { From be25681416ba946aaf1dd53037c6053d02608115 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Wed, 20 Nov 2024 14:21:58 +0200 Subject: [PATCH 34/71] Improve KafkaEdgeGrpcSession --- .../server/service/edge/rpc/KafkaEdgeGrpcSession.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java index 15a87ec3f1..e9a33520ad 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java @@ -44,7 +44,7 @@ import java.util.function.BiConsumer; @Slf4j public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession { - private final TbQueueConsumer> edgeEventsConsumer; + private final TbCoreQueueFactory tbCoreQueueFactory; private volatile boolean isHighPriorityProcessing; @@ -57,7 +57,7 @@ public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession { BiConsumer sessionCloseListener, ScheduledExecutorService sendDownlinkExecutorService, int maxInboundMessageSize, int maxHighPriorityQueueSizePerSession) { super(ctx, outputStream, sessionOpenListener, sessionCloseListener, sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession); - edgeEventsConsumer = tbCoreQueueFactory.createEdgeEventMsgConsumer(tenantId, edge.getId()); + this.tbCoreQueueFactory = tbCoreQueueFactory; } private void processMsgs(List> msgs, TbQueueConsumer> consumer) { @@ -102,7 +102,7 @@ public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession { .name("TB Edge events") .msgPackProcessor(this::processMsgs) .pollInterval(ctx.getEdgeEventStorageSettings().getNoRecordsSleepInterval()) - .consumerCreator(() -> edgeEventsConsumer) + .consumerCreator(() -> tbCoreQueueFactory.createEdgeEventMsgConsumer(tenantId, edge.getId())) .consumerExecutor(consumerExecutor) .threadPrefix("edge-events") .build(); From b0c89f2931b9362926053ba397ab439df8041aef Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 20 Nov 2024 19:09:02 +0100 Subject: [PATCH 35/71] Entity Data Query optimization - entityFilter.isFetchLastLevelOnly() to hit index and avoid seq scan. (and nr.relation_type_group = 'COMMON') --- .../server/dao/sql/query/DefaultEntityQueryRepository.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index dd1b4278d7..36ae779edd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -663,7 +663,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { .append("nr.").append(fromOrTo).append("_id").append(" = re.").append(toOrFrom).append("_id") .append(" and ") .append("nr.").append(fromOrTo).append("_type").append(" = re.").append(toOrFrom).append("_type"); - + notExistsPart.append(" and nr.relation_type_group = 'COMMON'"); // hit the index, the same condition are on the recursive query notExistsPart.append(")"); whereFilter += " and ( r_int.lvl = " + entityFilter.getMaxLevel() + " OR " + notExistsPart.toString() + ")"; } @@ -755,7 +755,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { .append("nr.").append(fromOrTo).append("_type").append(" = re.").append(toOrFrom).append("_type") .append(" and ") .append(whereFilter.toString().replaceAll("re\\.", "nr\\.")); - + notExistsPart.append(" and nr.relation_type_group = 'COMMON'"); // hit the index, the same condition are on the recursive query notExistsPart.append(")"); whereFilter.append(" and ( r_int.lvl = ").append(entityFilter.getMaxLevel()).append(" OR ").append(notExistsPart.toString()).append(")"); } From 1397ce80a02ee85794957ac8e39b332dff297cad Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 8 Nov 2024 08:36:09 +0100 Subject: [PATCH 36/71] set bigger timeout for getting resutl in tests --- .../timeseries/sql/LatestTimeseriesPerformanceTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/sql/LatestTimeseriesPerformanceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/sql/LatestTimeseriesPerformanceTest.java index d81eea997c..a2abfc28b6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/sql/LatestTimeseriesPerformanceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/sql/LatestTimeseriesPerformanceTest.java @@ -56,7 +56,8 @@ public class LatestTimeseriesPerformanceTest extends AbstractServiceTest { private static final String LONG_KEY = "longKey"; private static final String DOUBLE_KEY = "doubleKey"; private static final String BOOLEAN_KEY = "booleanKey"; - public static final int AMOUNT_OF_UNIQ_KEY = 10000; + private static final int AMOUNT_OF_UNIQ_KEY = 10000; + private static final int TIMEOUT = 100; private final Random random = new Random(); @@ -102,7 +103,7 @@ public class LatestTimeseriesPerformanceTest extends AbstractServiceTest { futures.add(save(generateDblEntry(getRandomKey()))); futures.add(save(generateBoolEntry(getRandomKey()))); } - Futures.allAsList(futures).get(60, TimeUnit.SECONDS); + Futures.allAsList(futures).get(TIMEOUT, TimeUnit.SECONDS); long endTime = System.currentTimeMillis(); long totalTime = endTime - startTime; @@ -120,7 +121,7 @@ public class LatestTimeseriesPerformanceTest extends AbstractServiceTest { futures.add(save(generateDblEntry(i))); futures.add(save(generateBoolEntry(i))); } - Futures.allAsList(futures).get(60, TimeUnit.SECONDS); + Futures.allAsList(futures).get(TIMEOUT, TimeUnit.SECONDS); } private ListenableFuture save(TsKvEntry tsKvEntry) { From 8c16f8c2da46be1b4c4b4c61364dbb62c5c488d8 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Thu, 21 Nov 2024 11:53:56 +0200 Subject: [PATCH 37/71] Finish processing hp events --- .../server/service/edge/rpc/KafkaEdgeGrpcSession.java | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java index e9a33520ad..a65e6ab85f 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java @@ -116,6 +116,7 @@ public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession { public void processHighPriorityEvents() { isHighPriorityProcessing = true; super.processHighPriorityEvents(); + isHighPriorityProcessing = false; } @Override From c388034d01543ab3e11471c4219158802aec8a68 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 22 Nov 2024 10:26:06 +0200 Subject: [PATCH 38/71] Delete EdgeGrpcSession interface: use abstract instead --- .../edge/rpc/AbstractEdgeGrpcSession.java | 939 ------------------ .../service/edge/rpc/EdgeGrpcService.java | 48 +- .../service/edge/rpc/EdgeGrpcSession.java | 924 ++++++++++++++++- .../edge/rpc/KafkaEdgeGrpcSession.java | 28 +- .../edge/rpc/PostgresEdgeGrpcSession.java | 7 +- .../entitiy/EntityStateSourcingListener.java | 26 +- .../ttl/KafkaEdgeTopicsCleanUpService.java | 20 +- .../TbTransportQueueProducerProvider.java | 4 +- 8 files changed, 983 insertions(+), 1013 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java deleted file mode 100644 index 9fd94b4988..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/AbstractEdgeGrpcSession.java +++ /dev/null @@ -1,939 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc; - -import com.datastax.oss.driver.api.core.uuid.Uuids; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.SettableFuture; -import io.grpc.stub.StreamObserver; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.checkerframework.checker.nullness.qual.Nullable; -import org.springframework.data.util.Pair; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.EdgeUtils; -import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.edge.EdgeEvent; -import org.thingsboard.server.common.data.id.EdgeId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.limit.LimitedApi; -import org.thingsboard.server.common.data.notification.rule.trigger.EdgeCommunicationFailureTrigger; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.page.SortOrder; -import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; -import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; -import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; -import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; -import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; -import org.thingsboard.server.gen.edge.v1.AttributesRequestMsg; -import org.thingsboard.server.gen.edge.v1.ConnectRequestMsg; -import org.thingsboard.server.gen.edge.v1.ConnectResponseCode; -import org.thingsboard.server.gen.edge.v1.ConnectResponseMsg; -import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DeviceCredentialsRequestMsg; -import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DeviceRpcCallMsg; -import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; -import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import org.thingsboard.server.gen.edge.v1.DownlinkResponseMsg; -import org.thingsboard.server.gen.edge.v1.EdgeConfiguration; -import org.thingsboard.server.gen.edge.v1.EdgeUpdateMsg; -import org.thingsboard.server.gen.edge.v1.EdgeVersion; -import org.thingsboard.server.gen.edge.v1.EntityDataProto; -import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; -import org.thingsboard.server.gen.edge.v1.EntityViewsRequestMsg; -import org.thingsboard.server.gen.edge.v1.RelationRequestMsg; -import org.thingsboard.server.gen.edge.v1.RelationUpdateMsg; -import org.thingsboard.server.gen.edge.v1.RequestMsg; -import org.thingsboard.server.gen.edge.v1.RequestMsgType; -import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; -import org.thingsboard.server.gen.edge.v1.ResponseMsg; -import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg; -import org.thingsboard.server.gen.edge.v1.SyncCompletedMsg; -import org.thingsboard.server.gen.edge.v1.UplinkMsg; -import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; -import org.thingsboard.server.gen.edge.v1.UserCredentialsRequestMsg; -import org.thingsboard.server.gen.edge.v1.WidgetBundleTypesRequestMsg; -import org.thingsboard.server.service.edge.EdgeContextComponent; -import org.thingsboard.server.service.edge.rpc.fetch.EdgeEventFetcher; -import org.thingsboard.server.service.edge.rpc.fetch.GeneralEdgeEventFetcher; -import org.thingsboard.server.service.edge.rpc.processor.alarm.AlarmProcessor; -import org.thingsboard.server.service.edge.rpc.processor.asset.AssetProcessor; -import org.thingsboard.server.service.edge.rpc.processor.asset.profile.AssetProfileProcessor; -import org.thingsboard.server.service.edge.rpc.processor.dashboard.DashboardProcessor; -import org.thingsboard.server.service.edge.rpc.processor.device.DeviceProcessor; -import org.thingsboard.server.service.edge.rpc.processor.device.profile.DeviceProfileProcessor; -import org.thingsboard.server.service.edge.rpc.processor.entityview.EntityViewProcessor; -import org.thingsboard.server.service.edge.rpc.processor.relation.RelationProcessor; -import org.thingsboard.server.service.edge.rpc.processor.resource.ResourceProcessor; - -import java.io.Closeable; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.BiConsumer; - -@Slf4j -@Data -public abstract class AbstractEdgeGrpcSession implements EdgeGrpcSession, Closeable { - - private static final String QUEUE_START_TS_ATTR_KEY = "queueStartTs"; - private static final String QUEUE_START_SEQ_ID_ATTR_KEY = "queueStartSeqId"; - - private static final int MAX_DOWNLINK_ATTEMPTS = 10; - private static final String RATE_LIMIT_REACHED = "Rate limit reached"; - - protected static final ConcurrentLinkedQueue highPriorityQueue = new ConcurrentLinkedQueue<>(); - - protected UUID sessionId; - private BiConsumer sessionOpenListener; - private BiConsumer sessionCloseListener; - - private final EdgeSessionState sessionState = new EdgeSessionState(); - private final ReentrantLock downlinkMsgLock = new ReentrantLock(); - - protected EdgeContextComponent ctx; - protected Edge edge; - protected TenantId tenantId; - - private Long newStartTs; - private Long previousStartTs; - private Long newStartSeqId; - private Long previousStartSeqId; - private Long seqIdEnd; - - private StreamObserver inputStream; - private StreamObserver outputStream; - - private boolean connected; - private volatile boolean syncCompleted; - - private EdgeVersion edgeVersion; - private int maxInboundMessageSize; - private int clientMaxInboundMessageSize; - private int maxHighPriorityQueueSizePerSession; - - private ScheduledExecutorService sendDownlinkExecutorService; - - public AbstractEdgeGrpcSession(EdgeContextComponent ctx, StreamObserver outputStream, - BiConsumer sessionOpenListener, - BiConsumer sessionCloseListener, - ScheduledExecutorService sendDownlinkExecutorService, - int maxInboundMessageSize, int maxHighPriorityQueueSizePerSession) { - this.sessionId = UUID.randomUUID(); - this.ctx = ctx; - this.outputStream = outputStream; - this.sessionOpenListener = sessionOpenListener; - this.sessionCloseListener = sessionCloseListener; - this.sendDownlinkExecutorService = sendDownlinkExecutorService; - this.maxInboundMessageSize = maxInboundMessageSize; - this.maxHighPriorityQueueSizePerSession = maxHighPriorityQueueSizePerSession; - initInputStream(); - } - - public void initInputStream() { - inputStream = new StreamObserver<>() { - @Override - public void onNext(RequestMsg requestMsg) { - if (!connected && requestMsg.getMsgType().equals(RequestMsgType.CONNECT_RPC_MESSAGE)) { - ConnectResponseMsg responseMsg = processConnect(requestMsg.getConnectRequestMsg()); - outputStream.onNext(ResponseMsg.newBuilder() - .setConnectResponseMsg(responseMsg) - .build()); - if (ConnectResponseCode.ACCEPTED != responseMsg.getResponseCode()) { - outputStream.onError(new RuntimeException(responseMsg.getErrorMsg())); - } else { - if (requestMsg.getConnectRequestMsg().hasMaxInboundMessageSize()) { - log.debug("[{}][{}] Client max inbound message size: {}", tenantId, sessionId, requestMsg.getConnectRequestMsg().getMaxInboundMessageSize()); - clientMaxInboundMessageSize = requestMsg.getConnectRequestMsg().getMaxInboundMessageSize(); - } - connected = true; - } - } - if (connected) { - if (requestMsg.getMsgType().equals(RequestMsgType.SYNC_REQUEST_RPC_MESSAGE)) { - if (requestMsg.hasSyncRequestMsg()) { - boolean fullSync = false; - if (requestMsg.getSyncRequestMsg().hasFullSync()) { - fullSync = requestMsg.getSyncRequestMsg().getFullSync(); - } - startSyncProcess(fullSync); - } else { - syncCompleted = true; - } - } - if (requestMsg.getMsgType().equals(RequestMsgType.UPLINK_RPC_MESSAGE)) { - if (requestMsg.hasUplinkMsg()) { - onUplinkMsg(requestMsg.getUplinkMsg()); - } - if (requestMsg.hasDownlinkResponseMsg()) { - onDownlinkResponse(requestMsg.getDownlinkResponseMsg()); - } - } - } - } - - @Override - public void onError(Throwable t) { - log.error("[{}][{}] Stream was terminated due to error:", tenantId, sessionId, t); - closeSession(); - } - - @Override - public void onCompleted() { - log.info("[{}][{}] Stream was closed and completed successfully!", tenantId, sessionId); - closeSession(); - } - - private void closeSession() { - connected = false; - if (edge != null) { - try { - sessionCloseListener.accept(edge, sessionId); - } catch (Exception ignored) { - } - } - try { - outputStream.onCompleted(); - } catch (Exception ignored) { - } - } - }; - } - - @Override - public void onConfigurationUpdate(Edge edge) { - log.debug("[{}] onConfigurationUpdate [{}]", sessionId, edge); - this.tenantId = edge.getTenantId(); - this.edge = edge; - EdgeUpdateMsg edgeConfig = EdgeUpdateMsg.newBuilder() - .setConfiguration(ctx.getEdgeMsgConstructor().constructEdgeConfiguration(edge)).build(); - ResponseMsg edgeConfigMsg = ResponseMsg.newBuilder() - .setEdgeUpdateMsg(edgeConfig) - .build(); - sendDownlinkMsg(edgeConfigMsg); - } - - @Override - public void startSyncProcess(boolean fullSync) { - log.info("[{}][{}][{}] Staring edge sync process", tenantId, edge.getId(), sessionId); - syncCompleted = false; - interruptGeneralProcessingOnSync(); - doSync(new EdgeSyncCursor(ctx, edge, fullSync)); - } - - private void doSync(EdgeSyncCursor cursor) { - if (cursor.hasNext()) { - EdgeEventFetcher next = cursor.getNext(); - log.info("[{}][{}] starting sync process, cursor current idx = {}, class = {}", - tenantId, edge.getId(), cursor.getCurrentIdx(), next.getClass().getSimpleName()); - ListenableFuture> future = startProcessingEdgeEvents(next); - Futures.addCallback(future, new FutureCallback<>() { - @Override - public void onSuccess(@Nullable Pair result) { - doSync(cursor); - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}][{}] Exception during sync process", tenantId, edge.getId(), t); - } - }, ctx.getGrpcCallbackExecutorService()); - } else { - log.info("[{}][{}] sync process completed", tenantId, edge.getId()); - DownlinkMsg syncCompleteDownlinkMsg = DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .setSyncCompletedMsg(SyncCompletedMsg.newBuilder().build()) - .build(); - Futures.addCallback(sendDownlinkMsgsPack(Collections.singletonList(syncCompleteDownlinkMsg)), new FutureCallback<>() { - @Override - public void onSuccess(Boolean isInterrupted) { - markSyncCompletedSendEdgeEventUpdate(); - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}][{}] Exception during sending sync complete", tenantId, edge.getId(), t); - markSyncCompletedSendEdgeEventUpdate(); - } - }, ctx.getGrpcCallbackExecutorService()); - } - } - - protected void processEdgeEvents(EdgeEventFetcher fetcher, PageLink pageLink, SettableFuture> result) { - try { - if (!highPriorityQueue.isEmpty()) { - processHighPriorityEvents(); - } - PageData pageData = fetcher.fetchEdgeEvents(edge.getTenantId(), edge, pageLink); - if (isConnected() && !pageData.getData().isEmpty()) { - log.trace("[{}][{}][{}] event(s) are going to be processed.", tenantId, sessionId, pageData.getData().size()); - List downlinkMsgsPack = convertToDownlinkMsgsPack(pageData.getData()); - Futures.addCallback(sendDownlinkMsgsPack(downlinkMsgsPack), new FutureCallback<>() { - @Override - public void onSuccess(@Nullable Boolean isInterrupted) { - if (Boolean.TRUE.equals(isInterrupted)) { - log.debug("[{}][{}][{}] Send downlink messages task was interrupted", tenantId, edge.getId(), sessionId); - result.set(null); - } else { - if (isConnected() && pageData.hasNext()) { - processEdgeEvents(fetcher, pageLink.nextPageLink(), result); - } else { - EdgeEvent latestEdgeEvent = pageData.getData().get(pageData.getData().size() - 1); - UUID idOffset = latestEdgeEvent.getUuidId(); - if (idOffset != null) { - Long newStartTs = Uuids.unixTimestamp(idOffset); - long newStartSeqId = latestEdgeEvent.getSeqId(); - result.set(Pair.of(newStartTs, newStartSeqId)); - } else { - result.set(null); - } - } - } - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}] Failed to send downlink msgs pack", sessionId, t); - result.setException(t); - } - }, ctx.getGrpcCallbackExecutorService()); - } else { - log.trace("[{}] no event(s) found. Stop processing edge events", sessionId); - result.set(null); - } - } catch (Exception e) { - log.error("[{}] Failed to fetch edge events", sessionId, e); - result.setException(e); - } - } - - private ConnectResponseMsg processConnect(ConnectRequestMsg request) { - log.trace("[{}] processConnect [{}]", sessionId, request); - Optional optional = ctx.getEdgeService().findEdgeByRoutingKey(TenantId.SYS_TENANT_ID, request.getEdgeRoutingKey()); - if (optional.isPresent()) { - edge = optional.get(); - tenantId = edge.getTenantId(); - try { - if (edge.getSecret().equals(request.getEdgeSecret())) { - sessionOpenListener.accept(edge.getId(), this); - edgeVersion = request.getEdgeVersion(); - processSaveEdgeVersionAsAttribute(request.getEdgeVersion().name()); - return ConnectResponseMsg.newBuilder() - .setResponseCode(ConnectResponseCode.ACCEPTED) - .setErrorMsg("") - .setConfiguration(ctx.getEdgeMsgConstructor().constructEdgeConfiguration(edge)) - .setMaxInboundMessageSize(maxInboundMessageSize) - .build(); - } - String error = "Failed to validate the edge!"; - String failureMsg = String.format("%s Provided request secret: %s", error, request.getEdgeSecret()); - ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) - .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(error).build()); - return ConnectResponseMsg.newBuilder() - .setResponseCode(ConnectResponseCode.BAD_CREDENTIALS) - .setErrorMsg(failureMsg) - .setConfiguration(EdgeConfiguration.getDefaultInstance()).build(); - } catch (Exception e) { - String failureMsg = "Failed to process edge connection!"; - ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) - .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(e.getMessage()).build()); - log.error(failureMsg, e); - return ConnectResponseMsg.newBuilder() - .setResponseCode(ConnectResponseCode.SERVER_UNAVAILABLE) - .setErrorMsg(failureMsg) - .setConfiguration(EdgeConfiguration.getDefaultInstance()).build(); - } - } - return ConnectResponseMsg.newBuilder() - .setResponseCode(ConnectResponseCode.BAD_CREDENTIALS) - .setErrorMsg("Failed to find the edge! Routing key: " + request.getEdgeRoutingKey()) - .setConfiguration(EdgeConfiguration.getDefaultInstance()).build(); - } - - private void processSaveEdgeVersionAsAttribute(String edgeVersion) { - AttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(new StringDataEntry(DataConstants.EDGE_VERSION_ATTR_KEY, edgeVersion), System.currentTimeMillis()); - ctx.getAttributesService().save(tenantId, edge.getId(), AttributeScope.SERVER_SCOPE, attributeKvEntry); - } - - private void interruptGeneralProcessingOnSync() { - log.debug("[{}][{}][{}] Sync process started. General processing interrupted!", tenantId, edge.getId(), sessionId); - stopCurrentSendDownlinkMsgsTask(true); - } - - protected ListenableFuture sendDownlinkMsgsPack(List downlinkMsgsPack) { - interruptPreviousSendDownlinkMsgsTask(); - - sessionState.setSendDownlinkMsgsFuture(SettableFuture.create()); - sessionState.getPendingMsgsMap().clear(); - - downlinkMsgsPack.forEach(msg -> sessionState.getPendingMsgsMap().put(msg.getDownlinkMsgId(), msg)); - scheduleDownlinkMsgsPackSend(1); - - return sessionState.getSendDownlinkMsgsFuture(); - } - - private void interruptPreviousSendDownlinkMsgsTask() { - if (sessionState.getSendDownlinkMsgsFuture() != null && !sessionState.getSendDownlinkMsgsFuture().isDone() - || sessionState.getScheduledSendDownlinkTask() != null && !sessionState.getScheduledSendDownlinkTask().isCancelled()) { - log.debug("[{}][{}][{}] Previous send downlink future was not properly completed, stopping it now!", tenantId, edge.getId(), sessionId); - stopCurrentSendDownlinkMsgsTask(true); - } - } - - private void onUplinkMsg(UplinkMsg uplinkMsg) { - if (isRateLimitViolated(uplinkMsg)) { - return; - } - ListenableFuture> future = processUplinkMsg(uplinkMsg); - Futures.addCallback(future, new FutureCallback<>() { - @Override - public void onSuccess(@Nullable List result) { - sendResponseMessage(uplinkMsg.getUplinkMsgId(), true, null); - } - - @Override - public void onFailure(Throwable t) { - String errorMsg = EdgeUtils.createErrorMsgFromRootCauseAndStackTrace(t); - sendResponseMessage(uplinkMsg.getUplinkMsgId(), false, errorMsg); - } - }, ctx.getGrpcCallbackExecutorService()); - } - - private boolean isRateLimitViolated(UplinkMsg uplinkMsg) { - if (!ctx.getRateLimitService().checkRateLimit(LimitedApi.EDGE_UPLINK_MESSAGES, tenantId) || - !ctx.getRateLimitService().checkRateLimit(LimitedApi.EDGE_UPLINK_MESSAGES_PER_EDGE, tenantId, edge.getId())) { - String errorMsg = String.format("Failed to process uplink message. %s", RATE_LIMIT_REACHED); - sendResponseMessage(uplinkMsg.getUplinkMsgId(), false, errorMsg); - return true; - } - return false; - } - - private void scheduleDownlinkMsgsPackSend(int attempt) { - Runnable sendDownlinkMsgsTask = () -> { - try { - if (isConnected() && !sessionState.getPendingMsgsMap().values().isEmpty()) { - List copy = new ArrayList<>(sessionState.getPendingMsgsMap().values()); - if (attempt > 1) { - String error = "Failed to deliver the batch"; - String failureMsg = String.format("{%s}: {%s}", error, copy); - if (attempt == 2) { - // Send a failure notification only on the second attempt. - // This ensures that failure alerts are sent just once to avoid redundant notifications. - ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId) - .edgeId(edge.getId()).customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(error).build()); - } - log.warn("[{}][{}] {}, attempt: {}", tenantId, sessionId, failureMsg, attempt); - } - log.trace("[{}][{}][{}] downlink msg(s) are going to be send.", tenantId, sessionId, copy.size()); - for (DownlinkMsg downlinkMsg : copy) { - if (clientMaxInboundMessageSize != 0 && downlinkMsg.getSerializedSize() > clientMaxInboundMessageSize) { - String error = String.format("Client max inbound message size %s is exceeded. Please increase value of CLOUD_RPC_MAX_INBOUND_MESSAGE_SIZE " + - "env variable on the edge and restart it.", clientMaxInboundMessageSize); - String message = String.format("Downlink msg size %s exceeds client max inbound message size %s. " + - "Please increase value of CLOUD_RPC_MAX_INBOUND_MESSAGE_SIZE env variable on the edge and restart it.", downlinkMsg.getSerializedSize(), clientMaxInboundMessageSize); - log.error("[{}][{}][{}] {} Message {}", tenantId, edge.getId(), sessionId, message, downlinkMsg); - ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId) - .edgeId(edge.getId()).customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(message).error(error).build()); - sessionState.getPendingMsgsMap().remove(downlinkMsg.getDownlinkMsgId()); - } else { - sendDownlinkMsg(ResponseMsg.newBuilder() - .setDownlinkMsg(downlinkMsg) - .build()); - } - } - if (attempt < MAX_DOWNLINK_ATTEMPTS) { - scheduleDownlinkMsgsPackSend(attempt + 1); - } else { - String failureMsg = String.format("Failed to deliver messages: %s", copy); - log.warn("[{}][{}] Failed to deliver the batch after {} attempts. Next messages are going to be discarded {}", - tenantId, sessionId, MAX_DOWNLINK_ATTEMPTS, copy); - ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) - .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg) - .error("Failed to deliver messages after " + MAX_DOWNLINK_ATTEMPTS + " attempts").build()); - stopCurrentSendDownlinkMsgsTask(false); - } - } else { - stopCurrentSendDownlinkMsgsTask(false); - } - } catch (Exception e) { - log.warn("[{}][{}] Failed to send downlink msgs. Error msg {}", tenantId, sessionId, e.getMessage(), e); - stopCurrentSendDownlinkMsgsTask(true); - } - }; - - if (attempt == 1) { - sendDownlinkExecutorService.submit(sendDownlinkMsgsTask); - } else { - sessionState.setScheduledSendDownlinkTask( - sendDownlinkExecutorService.schedule( - sendDownlinkMsgsTask, - ctx.getEdgeEventStorageSettings().getSleepIntervalBetweenBatches(), - TimeUnit.MILLISECONDS) - ); - } - } - - private void sendResponseMessage(int uplinkMsgId, boolean success, String errorMsg) { - UplinkResponseMsg.Builder responseBuilder = UplinkResponseMsg.newBuilder() - .setUplinkMsgId(uplinkMsgId) - .setSuccess(success); - if (errorMsg != null) { - responseBuilder.setErrorMsg(errorMsg); - } - sendDownlinkMsg(ResponseMsg.newBuilder() - .setUplinkResponseMsg(responseBuilder.build()) - .build()); - } - - private void onDownlinkResponse(DownlinkResponseMsg msg) { - try { - if (msg.getSuccess()) { - sessionState.getPendingMsgsMap().remove(msg.getDownlinkMsgId()); - log.debug("[{}][{}] Msg has been processed successfully! Msg Id: [{}], Msg: {}", tenantId, edge.getRoutingKey(), msg.getDownlinkMsgId(), msg); - } else { - log.error("[{}][{}] Msg processing failed! Msg Id: [{}], Error msg: {}", tenantId, edge.getRoutingKey(), msg.getDownlinkMsgId(), msg.getErrorMsg()); - } - if (sessionState.getPendingMsgsMap().isEmpty()) { - log.debug("[{}][{}] Pending msgs map is empty. Stopping current iteration", tenantId, edge.getRoutingKey()); - stopCurrentSendDownlinkMsgsTask(false); - } - } catch (Exception e) { - log.error("[{}][{}] Can't process downlink response message [{}]", tenantId, sessionId, msg, e); - } - } - - @Override - public void processHighPriorityEvents() { - try { - List highPriorityEvents = new ArrayList<>(); - EdgeEvent event; - while ((event = highPriorityQueue.poll()) != null) { - highPriorityEvents.add(event); - } - List downlinkMsgsPack = convertToDownlinkMsgsPack(highPriorityEvents); - sendDownlinkMsgsPack(downlinkMsgsPack).get(); - } catch (Exception e) { - log.error("[{}] Failed to process high priority events", sessionId, e); - } - } - - @Override - public ListenableFuture processEdgeEvents() throws Exception { - SettableFuture result = SettableFuture.create(); - log.trace("[{}][{}] starting processing edge events", tenantId, sessionId); - if (isConnected() && isSyncCompleted()) { - Pair startTsAndSeqId = getQueueStartTsAndSeqId().get(); - previousStartTs = startTsAndSeqId.getFirst(); - previousStartSeqId = startTsAndSeqId.getSecond(); - GeneralEdgeEventFetcher fetcher = new GeneralEdgeEventFetcher( - previousStartTs, - previousStartSeqId, - seqIdEnd, - false, - Integer.toUnsignedLong(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount()), - ctx.getEdgeEventService()); - Futures.addCallback(startProcessingEdgeEvents(fetcher), new FutureCallback<>() { - @Override - public void onSuccess(@Nullable Pair newStartTsAndSeqId) { - if (newStartTsAndSeqId != null) { - ListenableFuture> updateFuture = updateQueueStartTsAndSeqId(newStartTsAndSeqId); - Futures.addCallback(updateFuture, new FutureCallback<>() { - @Override - public void onSuccess(@Nullable List list) { - log.debug("[{}][{}] queue offset was updated [{}]", tenantId, sessionId, newStartTsAndSeqId); - if (fetcher.isSeqIdNewCycleStarted()) { - seqIdEnd = fetcher.getSeqIdEnd(); - boolean newEventsAvailable = isNewEdgeEventsAvailable(); - result.set(newEventsAvailable); - } else { - seqIdEnd = null; - boolean newEventsAvailable = isSeqIdStartedNewCycle(); - if (!newEventsAvailable) { - newEventsAvailable = isNewEdgeEventsAvailable(); - } - result.set(newEventsAvailable); - } - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}][{}] Failed to update queue offset [{}]", tenantId, sessionId, newStartTsAndSeqId, t); - result.setException(t); - } - }, ctx.getGrpcCallbackExecutorService()); - } else { - log.trace("[{}][{}] newStartTsAndSeqId is null. Skipping iteration without db update", tenantId, sessionId); - result.set(Boolean.FALSE); - } - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}][{}] Failed to process events", tenantId, sessionId, t); - result.setException(t); - } - }, ctx.getGrpcCallbackExecutorService()); - } else { - log.trace("[{}][{}] edge is not connected or sync is not completed. Skipping iteration", tenantId, sessionId); - result.set(null); - } - return result; - } - - protected List convertToDownlinkMsgsPack(List edgeEvents) { - List result = new ArrayList<>(); - for (EdgeEvent edgeEvent : edgeEvents) { - log.trace("[{}][{}] converting edge event to downlink msg [{}]", tenantId, sessionId, edgeEvent); - DownlinkMsg downlinkMsg = null; - try { - switch (edgeEvent.getAction()) { - case UPDATED, ADDED, DELETED, ASSIGNED_TO_EDGE, UNASSIGNED_FROM_EDGE, ALARM_ACK, ALARM_CLEAR, - ALARM_DELETE, CREDENTIALS_UPDATED, RELATION_ADD_OR_UPDATE, RELATION_DELETED, RPC_CALL, - ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER, ADDED_COMMENT, UPDATED_COMMENT, DELETED_COMMENT -> { - downlinkMsg = convertEntityEventToDownlink(edgeEvent); - if (downlinkMsg != null && downlinkMsg.getWidgetTypeUpdateMsgCount() > 0) { - log.trace("[{}][{}] widgetTypeUpdateMsg message processed, downlinkMsgId = {}", tenantId, sessionId, downlinkMsg.getDownlinkMsgId()); - } else { - log.trace("[{}][{}] entity message processed [{}]", tenantId, sessionId, downlinkMsg); - } - } - case ATTRIBUTES_UPDATED, POST_ATTRIBUTES, ATTRIBUTES_DELETED, TIMESERIES_UPDATED -> - downlinkMsg = ctx.getTelemetryProcessor().convertTelemetryEventToDownlink(edge, edgeEvent); - default -> log.warn("[{}][{}] Unsupported action type [{}]", tenantId, sessionId, edgeEvent.getAction()); - } - } catch (Exception e) { - log.error("[{}][{}] Exception during converting edge event to downlink msg", tenantId, sessionId, e); - } - if (downlinkMsg != null) { - result.add(downlinkMsg); - } - } - return result; - } - - private ListenableFuture> getQueueStartTsAndSeqId() { - ListenableFuture> future = - ctx.getAttributesService().find(edge.getTenantId(), edge.getId(), AttributeScope.SERVER_SCOPE, Arrays.asList(QUEUE_START_TS_ATTR_KEY, QUEUE_START_SEQ_ID_ATTR_KEY)); - return Futures.transform(future, attributeKvEntries -> { - long startTs = 0L; - long startSeqId = 0L; - for (AttributeKvEntry attributeKvEntry : attributeKvEntries) { - if (QUEUE_START_TS_ATTR_KEY.equals(attributeKvEntry.getKey())) { - startTs = attributeKvEntry.getLongValue().isPresent() ? attributeKvEntry.getLongValue().get() : 0L; - } - if (QUEUE_START_SEQ_ID_ATTR_KEY.equals(attributeKvEntry.getKey())) { - startSeqId = attributeKvEntry.getLongValue().isPresent() ? attributeKvEntry.getLongValue().get() : 0L; - } - } - if (startSeqId == 0L) { - startSeqId = findStartSeqIdFromOldestEventIfAny(); - } - return Pair.of(startTs, startSeqId); - }, ctx.getGrpcCallbackExecutorService()); - } - - private boolean isSeqIdStartedNewCycle() { - try { - TimePageLink pageLink = new TimePageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount(), 0, null, null, newStartTs, System.currentTimeMillis()); - PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), 0L, previousStartSeqId == 0 ? null : previousStartSeqId - 1, pageLink); - return !edgeEvents.getData().isEmpty(); - } catch (Exception e) { - log.error("[{}][{}][{}] Failed to execute isSeqIdStartedNewCycle", tenantId, edge.getId(), sessionId, e); - } - return false; - } - - private boolean isNewEdgeEventsAvailable() { - try { - TimePageLink pageLink = new TimePageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount(), 0, null, null, newStartTs, System.currentTimeMillis()); - PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), newStartSeqId, null, pageLink); - return !edgeEvents.getData().isEmpty() || !highPriorityQueue.isEmpty(); - } catch (Exception e) { - log.error("[{}][{}][{}] Failed to execute isNewEdgeEventsAvailable", tenantId, edge.getId(), sessionId, e); - } - return false; - } - - private long findStartSeqIdFromOldestEventIfAny() { - long startSeqId = 0L; - try { - TimePageLink pageLink = new TimePageLink(1, 0, null, new SortOrder("createdTime"), null, null); - PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), null, null, pageLink); - if (!edgeEvents.getData().isEmpty()) { - startSeqId = edgeEvents.getData().get(0).getSeqId() - 1; - } - } catch (Exception e) { - log.error("[{}][{}][{}] Failed to execute findStartSeqIdFromOldestEventIfAny", tenantId, edge.getId(), sessionId, e); - } - return startSeqId; - } - - private ListenableFuture> updateQueueStartTsAndSeqId(Pair pair) { - newStartTs = pair.getFirst(); - newStartSeqId = pair.getSecond(); - log.trace("[{}] updateQueueStartTsAndSeqId [{}][{}][{}]", sessionId, edge.getId(), newStartTs, newStartSeqId); - List attributes = Arrays.asList( - new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_TS_ATTR_KEY, newStartTs), System.currentTimeMillis()), - new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_SEQ_ID_ATTR_KEY, newStartSeqId), System.currentTimeMillis())); - return ctx.getAttributesService().save(edge.getTenantId(), edge.getId(), AttributeScope.SERVER_SCOPE, attributes); - } - - protected ListenableFuture> startProcessingEdgeEvents(EdgeEventFetcher fetcher) { - SettableFuture> result = SettableFuture.create(); - PageLink pageLink = fetcher.getPageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount()); - processEdgeEvents(fetcher, pageLink, result); - return result; - } - - private void markSyncCompletedSendEdgeEventUpdate() { - syncCompleted = true; - ctx.getClusterService().onEdgeEventUpdate(new EdgeEventUpdateMsg(edge.getTenantId(), edge.getId())); - } - - private void stopCurrentSendDownlinkMsgsTask(Boolean isInterrupted) { - if (sessionState.getSendDownlinkMsgsFuture() != null && !sessionState.getSendDownlinkMsgsFuture().isDone()) { - sessionState.getSendDownlinkMsgsFuture().set(isInterrupted); - } - if (sessionState.getScheduledSendDownlinkTask() != null) { - sessionState.getScheduledSendDownlinkTask().cancel(true); - } - } - - private void sendDownlinkMsg(ResponseMsg downlinkMsg) { - if (downlinkMsg.getDownlinkMsg().getWidgetTypeUpdateMsgCount() > 0) { - log.trace("[{}][{}] Sending downlink widgetTypeUpdateMsg, downlinkMsgId = {}", tenantId, sessionId, downlinkMsg.getDownlinkMsg().getDownlinkMsgId()); - } else { - log.trace("[{}][{}] Sending downlink msg [{}]", tenantId, sessionId, downlinkMsg); - } - if (isConnected()) { - downlinkMsgLock.lock(); - try { - outputStream.onNext(downlinkMsg); - } catch (Exception e) { - log.error("[{}][{}] Failed to send downlink message [{}]", tenantId, sessionId, downlinkMsg, e); - connected = false; - sessionCloseListener.accept(edge, sessionId); - } finally { - downlinkMsgLock.unlock(); - } - log.trace("[{}][{}] Response msg successfully sent. downlinkMsgId = {}", tenantId, sessionId, downlinkMsg.getDownlinkMsg().getDownlinkMsgId()); - } - } - - protected DownlinkMsg convertEntityEventToDownlink(EdgeEvent edgeEvent) { - log.trace("[{}] Executing convertEntityEventToDownlink, edgeEvent [{}], action [{}]", edgeEvent.getTenantId(), edgeEvent, edgeEvent.getAction()); - return switch (edgeEvent.getType()) { - case EDGE -> ctx.getEdgeProcessor().convertEdgeEventToDownlink(edgeEvent); - case DEVICE -> ctx.getDeviceProcessor().convertDeviceEventToDownlink(edgeEvent, edgeVersion); - case DEVICE_PROFILE -> ctx.getDeviceProfileProcessor().convertDeviceProfileEventToDownlink(edgeEvent, edgeVersion); - case ASSET_PROFILE -> ctx.getAssetProfileProcessor().convertAssetProfileEventToDownlink(edgeEvent, edgeVersion); - case ASSET -> ctx.getAssetProcessor().convertAssetEventToDownlink(edgeEvent, edgeVersion); - case ENTITY_VIEW -> ctx.getEntityViewProcessor().convertEntityViewEventToDownlink(edgeEvent, edgeVersion); - case DASHBOARD -> ctx.getDashboardProcessor().convertDashboardEventToDownlink(edgeEvent, edgeVersion); - case CUSTOMER -> ctx.getCustomerProcessor().convertCustomerEventToDownlink(edgeEvent, edgeVersion); - case RULE_CHAIN -> ctx.getRuleChainProcessor().convertRuleChainEventToDownlink(edgeEvent, edgeVersion); - case RULE_CHAIN_METADATA -> ctx.getRuleChainProcessor().convertRuleChainMetadataEventToDownlink(edgeEvent, edgeVersion); - case ALARM -> ctx.getAlarmProcessor().convertAlarmEventToDownlink(edgeEvent, edgeVersion); - case ALARM_COMMENT -> ctx.getAlarmProcessor().convertAlarmCommentEventToDownlink(edgeEvent, edgeVersion); - case USER -> ctx.getUserProcessor().convertUserEventToDownlink(edgeEvent, edgeVersion); - case RELATION -> ctx.getRelationProcessor().convertRelationEventToDownlink(edgeEvent, edgeVersion); - case WIDGETS_BUNDLE -> ctx.getWidgetBundleProcessor().convertWidgetsBundleEventToDownlink(edgeEvent, edgeVersion); - case WIDGET_TYPE -> ctx.getWidgetTypeProcessor().convertWidgetTypeEventToDownlink(edgeEvent, edgeVersion); - case ADMIN_SETTINGS -> ctx.getAdminSettingsProcessor().convertAdminSettingsEventToDownlink(edgeEvent, edgeVersion); - case OTA_PACKAGE -> ctx.getOtaPackageProcessor().convertOtaPackageEventToDownlink(edgeEvent, edgeVersion); - case TB_RESOURCE -> ctx.getResourceProcessor().convertResourceEventToDownlink(edgeEvent, edgeVersion); - case QUEUE -> ctx.getQueueProcessor().convertQueueEventToDownlink(edgeEvent, edgeVersion); - case TENANT -> ctx.getTenantProcessor().convertTenantEventToDownlink(edgeEvent, edgeVersion); - case TENANT_PROFILE -> ctx.getTenantProfileProcessor().convertTenantProfileEventToDownlink(edgeEvent, edgeVersion); - case NOTIFICATION_RULE -> ctx.getNotificationEdgeProcessor().convertNotificationRuleToDownlink(edgeEvent); - case NOTIFICATION_TARGET -> ctx.getNotificationEdgeProcessor().convertNotificationTargetToDownlink(edgeEvent); - case NOTIFICATION_TEMPLATE -> ctx.getNotificationEdgeProcessor().convertNotificationTemplateToDownlink(edgeEvent); - case OAUTH2_CLIENT -> ctx.getOAuth2EdgeProcessor().convertOAuth2ClientEventToDownlink(edgeEvent, edgeVersion); - case DOMAIN -> ctx.getOAuth2EdgeProcessor().convertOAuth2DomainEventToDownlink(edgeEvent, edgeVersion); - default -> { - log.warn("[{}] Unsupported edge event type [{}]", edgeEvent.getTenantId(), edgeEvent); - yield null; - } - }; - } - - public void addEventToHighPriorityQueue(EdgeEvent edgeEvent) { - while (highPriorityQueue.size() > maxHighPriorityQueueSizePerSession) { - EdgeEvent oldestHighPriority = highPriorityQueue.poll(); - if (oldestHighPriority != null) { - log.warn("[{}][{}][{}] High priority queue is full. Removing oldest high priority event from queue {}", - tenantId, edge.getId(), sessionId, oldestHighPriority); - } - } - highPriorityQueue.add(edgeEvent); - } - - protected ListenableFuture> processUplinkMsg(UplinkMsg uplinkMsg) { - List> result = new ArrayList<>(); - try { - if (uplinkMsg.getDeviceProfileUpdateMsgCount() > 0) { - for (DeviceProfileUpdateMsg deviceProfileUpdateMsg : uplinkMsg.getDeviceProfileUpdateMsgList()) { - result.add(((DeviceProfileProcessor) ctx.getDeviceProfileEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processDeviceProfileMsgFromEdge(edge.getTenantId(), edge, deviceProfileUpdateMsg)); - } - } - if (uplinkMsg.getDeviceUpdateMsgCount() > 0) { - for (DeviceUpdateMsg deviceUpdateMsg : uplinkMsg.getDeviceUpdateMsgList()) { - result.add(((DeviceProcessor) ctx.getDeviceEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processDeviceMsgFromEdge(edge.getTenantId(), edge, deviceUpdateMsg)); - } - } - if (uplinkMsg.getDeviceCredentialsUpdateMsgCount() > 0) { - for (DeviceCredentialsUpdateMsg deviceCredentialsUpdateMsg : uplinkMsg.getDeviceCredentialsUpdateMsgList()) { - result.add(((DeviceProcessor) ctx.getDeviceEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processDeviceCredentialsMsgFromEdge(edge.getTenantId(), edge.getId(), deviceCredentialsUpdateMsg)); - } - } - if (uplinkMsg.getAssetProfileUpdateMsgCount() > 0) { - for (AssetProfileUpdateMsg assetProfileUpdateMsg : uplinkMsg.getAssetProfileUpdateMsgList()) { - result.add(((AssetProfileProcessor) ctx.getAssetProfileEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processAssetProfileMsgFromEdge(edge.getTenantId(), edge, assetProfileUpdateMsg)); - } - } - if (uplinkMsg.getAssetUpdateMsgCount() > 0) { - for (AssetUpdateMsg assetUpdateMsg : uplinkMsg.getAssetUpdateMsgList()) { - result.add(((AssetProcessor) ctx.getAssetEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processAssetMsgFromEdge(edge.getTenantId(), edge, assetUpdateMsg)); - } - } - if (uplinkMsg.getEntityViewUpdateMsgCount() > 0) { - for (EntityViewUpdateMsg entityViewUpdateMsg : uplinkMsg.getEntityViewUpdateMsgList()) { - result.add(((EntityViewProcessor) ctx.getEntityViewProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processEntityViewMsgFromEdge(edge.getTenantId(), edge, entityViewUpdateMsg)); - } - } - if (uplinkMsg.getEntityDataCount() > 0) { - for (EntityDataProto entityData : uplinkMsg.getEntityDataList()) { - result.addAll(ctx.getTelemetryProcessor().processTelemetryMsg(edge.getTenantId(), entityData)); - } - } - if (uplinkMsg.getAlarmUpdateMsgCount() > 0) { - for (AlarmUpdateMsg alarmUpdateMsg : uplinkMsg.getAlarmUpdateMsgList()) { - result.add(((AlarmProcessor) ctx.getAlarmEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processAlarmMsgFromEdge(edge.getTenantId(), edge.getId(), alarmUpdateMsg)); - } - } - if (uplinkMsg.getAlarmCommentUpdateMsgCount() > 0) { - for (AlarmCommentUpdateMsg alarmCommentUpdateMsg : uplinkMsg.getAlarmCommentUpdateMsgList()) { - result.add(((AlarmProcessor) ctx.getAlarmEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processAlarmCommentMsgFromEdge(edge.getTenantId(), edge.getId(), alarmCommentUpdateMsg)); - } - } - if (uplinkMsg.getRelationUpdateMsgCount() > 0) { - for (RelationUpdateMsg relationUpdateMsg : uplinkMsg.getRelationUpdateMsgList()) { - result.add(((RelationProcessor) ctx.getRelationEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processRelationMsgFromEdge(edge.getTenantId(), edge, relationUpdateMsg)); - } - } - if (uplinkMsg.getDashboardUpdateMsgCount() > 0) { - for (DashboardUpdateMsg dashboardUpdateMsg : uplinkMsg.getDashboardUpdateMsgList()) { - result.add(((DashboardProcessor) ctx.getDashboardEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processDashboardMsgFromEdge(edge.getTenantId(), edge, dashboardUpdateMsg)); - } - } - if (uplinkMsg.getResourceUpdateMsgCount() > 0) { - for (ResourceUpdateMsg resourceUpdateMsg : uplinkMsg.getResourceUpdateMsgList()) { - result.add(((ResourceProcessor) ctx.getResourceEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processResourceMsgFromEdge(edge.getTenantId(), edge, resourceUpdateMsg)); - } - } - if (uplinkMsg.getRuleChainMetadataRequestMsgCount() > 0) { - for (RuleChainMetadataRequestMsg ruleChainMetadataRequestMsg : uplinkMsg.getRuleChainMetadataRequestMsgList()) { - result.add(ctx.getEdgeRequestsService().processRuleChainMetadataRequestMsg(edge.getTenantId(), edge, ruleChainMetadataRequestMsg)); - } - } - if (uplinkMsg.getAttributesRequestMsgCount() > 0) { - for (AttributesRequestMsg attributesRequestMsg : uplinkMsg.getAttributesRequestMsgList()) { - result.add(ctx.getEdgeRequestsService().processAttributesRequestMsg(edge.getTenantId(), edge, attributesRequestMsg)); - } - } - if (uplinkMsg.getRelationRequestMsgCount() > 0) { - for (RelationRequestMsg relationRequestMsg : uplinkMsg.getRelationRequestMsgList()) { - result.add(ctx.getEdgeRequestsService().processRelationRequestMsg(edge.getTenantId(), edge, relationRequestMsg)); - } - } - if (uplinkMsg.getUserCredentialsRequestMsgCount() > 0) { - for (UserCredentialsRequestMsg userCredentialsRequestMsg : uplinkMsg.getUserCredentialsRequestMsgList()) { - result.add(ctx.getEdgeRequestsService().processUserCredentialsRequestMsg(edge.getTenantId(), edge, userCredentialsRequestMsg)); - } - } - if (uplinkMsg.getDeviceCredentialsRequestMsgCount() > 0) { - for (DeviceCredentialsRequestMsg deviceCredentialsRequestMsg : uplinkMsg.getDeviceCredentialsRequestMsgList()) { - result.add(ctx.getEdgeRequestsService().processDeviceCredentialsRequestMsg(edge.getTenantId(), edge, deviceCredentialsRequestMsg)); - } - } - if (uplinkMsg.getDeviceRpcCallMsgCount() > 0) { - for (DeviceRpcCallMsg deviceRpcCallMsg : uplinkMsg.getDeviceRpcCallMsgList()) { - result.add(((DeviceProcessor) ctx.getDeviceEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) - .processDeviceRpcCallFromEdge(edge.getTenantId(), edge, deviceRpcCallMsg)); - } - } - if (uplinkMsg.getWidgetBundleTypesRequestMsgCount() > 0) { - for (WidgetBundleTypesRequestMsg widgetBundleTypesRequestMsg : uplinkMsg.getWidgetBundleTypesRequestMsgList()) { - result.add(ctx.getEdgeRequestsService().processWidgetBundleTypesRequestMsg(edge.getTenantId(), edge, widgetBundleTypesRequestMsg)); - } - } - if (uplinkMsg.getEntityViewsRequestMsgCount() > 0) { - for (EntityViewsRequestMsg entityViewRequestMsg : uplinkMsg.getEntityViewsRequestMsgList()) { - result.add(ctx.getEdgeRequestsService().processEntityViewsRequestMsg(edge.getTenantId(), edge, entityViewRequestMsg)); - } - } - } catch (Exception e) { - String failureMsg = String.format("Can't process uplink msg [%s] from edge", uplinkMsg); - log.error("[{}][{}] Can't process uplink msg [{}]", edge.getTenantId(), sessionId, uplinkMsg, e); - ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(edge.getTenantId()).edgeId(edge.getId()) - .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(e.getMessage()).build()); - return Futures.immediateFailedFuture(e); - } - return Futures.allAsList(result); - } - - @Override - public void close() { - log.debug("[{}][{}] Closing session", tenantId, sessionId); - connected = false; - try { - outputStream.onCompleted(); - } catch (Exception e) { - log.debug("[{}][{}] Failed to close output stream: {}", tenantId, sessionId, e.getMessage()); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java index 1b1a6de088..9c0aa101c4 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java @@ -58,6 +58,9 @@ import org.thingsboard.server.gen.edge.v1.EdgeRpcServiceGrpc; import org.thingsboard.server.gen.edge.v1.RequestMsg; import org.thingsboard.server.gen.edge.v1.ResponseMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeContextComponent; @@ -69,6 +72,7 @@ import java.io.InputStream; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -86,7 +90,7 @@ import java.util.function.Consumer; @TbCoreComponent public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase implements EdgeRpcService { - private final ConcurrentMap sessions = new ConcurrentHashMap<>(); + private final ConcurrentMap sessions = new ConcurrentHashMap<>(); private final ConcurrentMap sessionNewEventsLocks = new ConcurrentHashMap<>(); private final Map sessionNewEvents = new HashMap<>(); private final ConcurrentMap> sessionEdgeEventChecks = new ConcurrentHashMap<>(); @@ -120,9 +124,6 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i @Value("${edges.max_high_priority_queue_size_per_session:10000}") private int maxHighPriorityQueueSizePerSession; - @Value("#{'${queue.type:null}' == 'kafka'}") - private boolean isKafkaSupported; - @Autowired @Lazy private EdgeContextComponent ctx; @@ -139,9 +140,18 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i @Autowired private TbTransactionalCache edgeIdServiceIdCache; + @Autowired + private TopicService topicService; + @Autowired private TbCoreQueueFactory tbCoreQueueFactory; + @Autowired + private Optional kafkaSettings; + + @Autowired + private Optional kafkaTopicConfigs; + private Server server; private ScheduledExecutorService edgeEventProcessingExecutorService; @@ -210,13 +220,13 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i @Override public StreamObserver handleMsgs(StreamObserver outputStream) { - AbstractEdgeGrpcSession session = createEdgeGrpcSession(outputStream); + EdgeGrpcSession session = createEdgeGrpcSession(outputStream); return session.getInputStream(); } - private AbstractEdgeGrpcSession createEdgeGrpcSession(StreamObserver outputStream) { - return isKafkaSupported - ? new KafkaEdgeGrpcSession(ctx, tbCoreQueueFactory, outputStream, this::onEdgeConnect, this::onEdgeDisconnect, + private EdgeGrpcSession createEdgeGrpcSession(StreamObserver outputStream) { + return kafkaSettings.isPresent() && kafkaTopicConfigs.isPresent() + ? new KafkaEdgeGrpcSession(ctx, topicService, tbCoreQueueFactory, kafkaSettings.get(), kafkaTopicConfigs.get(), outputStream, this::onEdgeConnect, this::onEdgeDisconnect, sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession) : new PostgresEdgeGrpcSession(ctx, outputStream, this::onEdgeConnect, this::onEdgeDisconnect, sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession); @@ -250,7 +260,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i @Override public void updateEdge(TenantId tenantId, Edge edge) { - AbstractEdgeGrpcSession session = sessions.get(edge.getId()); + EdgeGrpcSession session = sessions.get(edge.getId()); if (session != null && session.isConnected()) { log.debug("[{}] Updating configuration for edge [{}] [{}]", tenantId, edge.getName(), edge.getId()); session.onConfigurationUpdate(edge); @@ -261,9 +271,11 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i @Override public void deleteEdge(TenantId tenantId, EdgeId edgeId) { - AbstractEdgeGrpcSession session = sessions.get(edgeId); + EdgeGrpcSession session = sessions.get(edgeId); if (session != null && session.isConnected()) { log.info("[{}] Closing and removing session for edge [{}]", tenantId, edgeId); + session.destroy(); + session.deleteTopic(edgeId); session.close(); sessions.remove(edgeId); final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock()); @@ -278,7 +290,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } private void onEdgeEventUpdate(TenantId tenantId, EdgeId edgeId) { - AbstractEdgeGrpcSession session = sessions.get(edgeId); + EdgeGrpcSession session = sessions.get(edgeId); if (session != null && session.isConnected()) { log.trace("[{}] onEdgeEventUpdate [{}]", tenantId, edgeId.getId()); updateSessionEventsFlag(tenantId, edgeId); @@ -289,7 +301,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i TenantId tenantId = msg.getTenantId(); EdgeEvent edgeEvent = msg.getEdgeEvent(); EdgeId edgeId = edgeEvent.getEdgeId(); - AbstractEdgeGrpcSession session = sessions.get(edgeId); + EdgeGrpcSession session = sessions.get(edgeId); if (session != null && session.isConnected()) { log.trace("[{}] onEdgeEvent [{}]", tenantId, edgeId); session.addEventToHighPriorityQueue(edgeEvent); @@ -310,7 +322,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } } - private void onEdgeConnect(EdgeId edgeId, AbstractEdgeGrpcSession edgeGrpcSession) { + private void onEdgeConnect(EdgeId edgeId, EdgeGrpcSession edgeGrpcSession) { Edge edge = edgeGrpcSession.getEdge(); TenantId tenantId = edge.getTenantId(); log.info("[{}][{}] edge [{}] connected successfully.", tenantId, edgeGrpcSession.getSessionId(), edgeId); @@ -333,7 +345,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } private void startSyncProcess(TenantId tenantId, EdgeId edgeId, UUID requestId, String requestServiceId) { - AbstractEdgeGrpcSession session = sessions.get(edgeId); + EdgeGrpcSession session = sessions.get(edgeId); if (session != null) { if (!session.isSyncCompleted()) { clusterService.pushEdgeSyncResponseToCore(new FromEdgeSyncResponse(requestId, tenantId, edgeId, false, "Sync process is active at the moment"), requestServiceId); @@ -353,7 +365,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i ToEdgeSyncRequest request = new ToEdgeSyncRequest(UUID.randomUUID(), tenantId, edgeId, serviceInfoProvider.getServiceId()); UUID requestId = request.getId(); - AbstractEdgeGrpcSession session = sessions.get(request.getEdgeId()); + EdgeGrpcSession session = sessions.get(request.getEdgeId()); if (session != null && !session.isSyncCompleted()) { responseConsumer.accept(new FromEdgeSyncResponse(requestId, request.getTenantId(), request.getEdgeId(), false, "Sync process is active at the moment")); } else { @@ -387,7 +399,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } } - private void scheduleEdgeEventsCheck(AbstractEdgeGrpcSession session) { + private void scheduleEdgeEventsCheck(EdgeGrpcSession session) { EdgeId edgeId = session.getEdge().getId(); TenantId tenantId = session.getEdge().getTenantId(); if (sessions.containsKey(edgeId)) { @@ -434,7 +446,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } } - private void processEdgeEventMigrationIfNeeded(AbstractEdgeGrpcSession session, EdgeId edgeId) throws Exception { + private void processEdgeEventMigrationIfNeeded(EdgeGrpcSession session, EdgeId edgeId) throws Exception { boolean isMigrationProcessed = edgeEventsProcessed.getOrDefault(edgeId, Boolean.FALSE); if (!isMigrationProcessed) { Boolean eventsExist = session.migrateEdgeEvents().get(); @@ -463,7 +475,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i private void onEdgeDisconnect(Edge edge, UUID sessionId) { EdgeId edgeId = edge.getId(); log.info("[{}][{}] edge disconnected!", edgeId, sessionId); - AbstractEdgeGrpcSession toRemove = sessions.get(edgeId); + EdgeGrpcSession toRemove = sessions.get(edgeId); if (toRemove.getSessionId().equals(sessionId)) { toRemove = sessions.remove(edgeId); final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock()); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index 880712d6b2..4ad9c04353 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -15,23 +15,931 @@ */ package org.thingsboard.server.service.edge.rpc; +import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import io.grpc.stub.StreamObserver; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.springframework.data.util.Pair; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.limit.LimitedApi; +import org.thingsboard.server.common.data.notification.rule.trigger.EdgeCommunicationFailureTrigger; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.SortOrder; +import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; +import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; +import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; +import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; +import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; +import org.thingsboard.server.gen.edge.v1.AttributesRequestMsg; +import org.thingsboard.server.gen.edge.v1.ConnectRequestMsg; +import org.thingsboard.server.gen.edge.v1.ConnectResponseCode; +import org.thingsboard.server.gen.edge.v1.ConnectResponseMsg; +import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; +import org.thingsboard.server.gen.edge.v1.DeviceCredentialsRequestMsg; +import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; +import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; +import org.thingsboard.server.gen.edge.v1.DeviceRpcCallMsg; +import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; +import org.thingsboard.server.gen.edge.v1.DownlinkMsg; +import org.thingsboard.server.gen.edge.v1.DownlinkResponseMsg; +import org.thingsboard.server.gen.edge.v1.EdgeConfiguration; +import org.thingsboard.server.gen.edge.v1.EdgeUpdateMsg; +import org.thingsboard.server.gen.edge.v1.EdgeVersion; +import org.thingsboard.server.gen.edge.v1.EntityDataProto; +import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; +import org.thingsboard.server.gen.edge.v1.EntityViewsRequestMsg; +import org.thingsboard.server.gen.edge.v1.RelationRequestMsg; +import org.thingsboard.server.gen.edge.v1.RelationUpdateMsg; +import org.thingsboard.server.gen.edge.v1.RequestMsg; +import org.thingsboard.server.gen.edge.v1.RequestMsgType; +import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; +import org.thingsboard.server.gen.edge.v1.ResponseMsg; +import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg; +import org.thingsboard.server.gen.edge.v1.SyncCompletedMsg; +import org.thingsboard.server.gen.edge.v1.UplinkMsg; +import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; +import org.thingsboard.server.gen.edge.v1.UserCredentialsRequestMsg; +import org.thingsboard.server.gen.edge.v1.WidgetBundleTypesRequestMsg; +import org.thingsboard.server.service.edge.EdgeContextComponent; +import org.thingsboard.server.service.edge.rpc.fetch.EdgeEventFetcher; +import org.thingsboard.server.service.edge.rpc.fetch.GeneralEdgeEventFetcher; +import org.thingsboard.server.service.edge.rpc.processor.alarm.AlarmProcessor; +import org.thingsboard.server.service.edge.rpc.processor.asset.AssetProcessor; +import org.thingsboard.server.service.edge.rpc.processor.asset.profile.AssetProfileProcessor; +import org.thingsboard.server.service.edge.rpc.processor.dashboard.DashboardProcessor; +import org.thingsboard.server.service.edge.rpc.processor.device.DeviceProcessor; +import org.thingsboard.server.service.edge.rpc.processor.device.profile.DeviceProfileProcessor; +import org.thingsboard.server.service.edge.rpc.processor.entityview.EntityViewProcessor; +import org.thingsboard.server.service.edge.rpc.processor.relation.RelationProcessor; +import org.thingsboard.server.service.edge.rpc.processor.resource.ResourceProcessor; -public interface EdgeGrpcSession { +import java.io.Closeable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.BiConsumer; - void onConfigurationUpdate(Edge edge); +@Slf4j +@Data +public abstract class EdgeGrpcSession implements Closeable { - void startSyncProcess(boolean fullSync); + private static final String QUEUE_START_TS_ATTR_KEY = "queueStartTs"; + private static final String QUEUE_START_SEQ_ID_ATTR_KEY = "queueStartSeqId"; - boolean isConnected(); + private static final int MAX_DOWNLINK_ATTEMPTS = 10; + private static final String RATE_LIMIT_REACHED = "Rate limit reached"; - void destroy(); + protected static final ConcurrentLinkedQueue highPriorityQueue = new ConcurrentLinkedQueue<>(); - ListenableFuture migrateEdgeEvents() throws Exception; + protected UUID sessionId; + private BiConsumer sessionOpenListener; + private BiConsumer sessionCloseListener; - ListenableFuture processEdgeEvents() throws Exception; + private final EdgeSessionState sessionState = new EdgeSessionState(); + private final ReentrantLock downlinkMsgLock = new ReentrantLock(); - void processHighPriorityEvents(); + protected EdgeContextComponent ctx; + protected Edge edge; + protected TenantId tenantId; + + private Long newStartTs; + private Long previousStartTs; + private Long newStartSeqId; + private Long previousStartSeqId; + private Long seqIdEnd; + + private StreamObserver inputStream; + private StreamObserver outputStream; + + private boolean connected; + private volatile boolean syncCompleted; + + private EdgeVersion edgeVersion; + private int maxInboundMessageSize; + private int clientMaxInboundMessageSize; + private int maxHighPriorityQueueSizePerSession; + + private ScheduledExecutorService sendDownlinkExecutorService; + + public EdgeGrpcSession(EdgeContextComponent ctx, StreamObserver outputStream, + BiConsumer sessionOpenListener, + BiConsumer sessionCloseListener, + ScheduledExecutorService sendDownlinkExecutorService, + int maxInboundMessageSize, int maxHighPriorityQueueSizePerSession) { + this.sessionId = UUID.randomUUID(); + this.ctx = ctx; + this.outputStream = outputStream; + this.sessionOpenListener = sessionOpenListener; + this.sessionCloseListener = sessionCloseListener; + this.sendDownlinkExecutorService = sendDownlinkExecutorService; + this.maxInboundMessageSize = maxInboundMessageSize; + this.maxHighPriorityQueueSizePerSession = maxHighPriorityQueueSizePerSession; + initInputStream(); + } + + protected abstract ListenableFuture migrateEdgeEvents() throws Exception; + + public void initInputStream() { + inputStream = new StreamObserver<>() { + @Override + public void onNext(RequestMsg requestMsg) { + if (!connected && requestMsg.getMsgType().equals(RequestMsgType.CONNECT_RPC_MESSAGE)) { + ConnectResponseMsg responseMsg = processConnect(requestMsg.getConnectRequestMsg()); + outputStream.onNext(ResponseMsg.newBuilder() + .setConnectResponseMsg(responseMsg) + .build()); + if (ConnectResponseCode.ACCEPTED != responseMsg.getResponseCode()) { + outputStream.onError(new RuntimeException(responseMsg.getErrorMsg())); + } else { + if (requestMsg.getConnectRequestMsg().hasMaxInboundMessageSize()) { + log.debug("[{}][{}] Client max inbound message size: {}", tenantId, sessionId, requestMsg.getConnectRequestMsg().getMaxInboundMessageSize()); + clientMaxInboundMessageSize = requestMsg.getConnectRequestMsg().getMaxInboundMessageSize(); + } + connected = true; + } + } + if (connected) { + if (requestMsg.getMsgType().equals(RequestMsgType.SYNC_REQUEST_RPC_MESSAGE)) { + if (requestMsg.hasSyncRequestMsg()) { + boolean fullSync = false; + if (requestMsg.getSyncRequestMsg().hasFullSync()) { + fullSync = requestMsg.getSyncRequestMsg().getFullSync(); + } + startSyncProcess(fullSync); + } else { + syncCompleted = true; + } + } + if (requestMsg.getMsgType().equals(RequestMsgType.UPLINK_RPC_MESSAGE)) { + if (requestMsg.hasUplinkMsg()) { + onUplinkMsg(requestMsg.getUplinkMsg()); + } + if (requestMsg.hasDownlinkResponseMsg()) { + onDownlinkResponse(requestMsg.getDownlinkResponseMsg()); + } + } + } + } + + @Override + public void onError(Throwable t) { + log.error("[{}][{}] Stream was terminated due to error:", tenantId, sessionId, t); + closeSession(); + } + + @Override + public void onCompleted() { + log.info("[{}][{}] Stream was closed and completed successfully!", tenantId, sessionId); + closeSession(); + } + + private void closeSession() { + connected = false; + if (edge != null) { + try { + sessionCloseListener.accept(edge, sessionId); + } catch (Exception ignored) { + } + } + try { + outputStream.onCompleted(); + } catch (Exception ignored) { + } + } + }; + } + + public void onConfigurationUpdate(Edge edge) { + log.debug("[{}] onConfigurationUpdate [{}]", sessionId, edge); + this.tenantId = edge.getTenantId(); + this.edge = edge; + EdgeUpdateMsg edgeConfig = EdgeUpdateMsg.newBuilder() + .setConfiguration(ctx.getEdgeMsgConstructor().constructEdgeConfiguration(edge)).build(); + ResponseMsg edgeConfigMsg = ResponseMsg.newBuilder() + .setEdgeUpdateMsg(edgeConfig) + .build(); + sendDownlinkMsg(edgeConfigMsg); + } + + public void startSyncProcess(boolean fullSync) { + log.info("[{}][{}][{}] Staring edge sync process", tenantId, edge.getId(), sessionId); + syncCompleted = false; + interruptGeneralProcessingOnSync(); + doSync(new EdgeSyncCursor(ctx, edge, fullSync)); + } + + private void doSync(EdgeSyncCursor cursor) { + if (cursor.hasNext()) { + EdgeEventFetcher next = cursor.getNext(); + log.info("[{}][{}] starting sync process, cursor current idx = {}, class = {}", + tenantId, edge.getId(), cursor.getCurrentIdx(), next.getClass().getSimpleName()); + ListenableFuture> future = startProcessingEdgeEvents(next); + Futures.addCallback(future, new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Pair result) { + doSync(cursor); + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}][{}] Exception during sync process", tenantId, edge.getId(), t); + } + }, ctx.getGrpcCallbackExecutorService()); + } else { + log.info("[{}][{}] sync process completed", tenantId, edge.getId()); + DownlinkMsg syncCompleteDownlinkMsg = DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .setSyncCompletedMsg(SyncCompletedMsg.newBuilder().build()) + .build(); + Futures.addCallback(sendDownlinkMsgsPack(Collections.singletonList(syncCompleteDownlinkMsg)), new FutureCallback<>() { + @Override + public void onSuccess(Boolean isInterrupted) { + markSyncCompletedSendEdgeEventUpdate(); + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}][{}] Exception during sending sync complete", tenantId, edge.getId(), t); + markSyncCompletedSendEdgeEventUpdate(); + } + }, ctx.getGrpcCallbackExecutorService()); + } + } + + protected void processEdgeEvents(EdgeEventFetcher fetcher, PageLink pageLink, SettableFuture> result) { + try { + if (!highPriorityQueue.isEmpty()) { + processHighPriorityEvents(); + } + PageData pageData = fetcher.fetchEdgeEvents(edge.getTenantId(), edge, pageLink); + if (isConnected() && !pageData.getData().isEmpty()) { + log.trace("[{}][{}][{}] event(s) are going to be processed.", tenantId, sessionId, pageData.getData().size()); + List downlinkMsgsPack = convertToDownlinkMsgsPack(pageData.getData()); + Futures.addCallback(sendDownlinkMsgsPack(downlinkMsgsPack), new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Boolean isInterrupted) { + if (Boolean.TRUE.equals(isInterrupted)) { + log.debug("[{}][{}][{}] Send downlink messages task was interrupted", tenantId, edge.getId(), sessionId); + result.set(null); + } else { + if (isConnected() && pageData.hasNext()) { + processEdgeEvents(fetcher, pageLink.nextPageLink(), result); + } else { + EdgeEvent latestEdgeEvent = pageData.getData().get(pageData.getData().size() - 1); + UUID idOffset = latestEdgeEvent.getUuidId(); + if (idOffset != null) { + Long newStartTs = Uuids.unixTimestamp(idOffset); + long newStartSeqId = latestEdgeEvent.getSeqId(); + result.set(Pair.of(newStartTs, newStartSeqId)); + } else { + result.set(null); + } + } + } + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}] Failed to send downlink msgs pack", sessionId, t); + result.setException(t); + } + }, ctx.getGrpcCallbackExecutorService()); + } else { + log.trace("[{}] no event(s) found. Stop processing edge events", sessionId); + result.set(null); + } + } catch (Exception e) { + log.error("[{}] Failed to fetch edge events", sessionId, e); + result.setException(e); + } + } + + private ConnectResponseMsg processConnect(ConnectRequestMsg request) { + log.trace("[{}] processConnect [{}]", sessionId, request); + Optional optional = ctx.getEdgeService().findEdgeByRoutingKey(TenantId.SYS_TENANT_ID, request.getEdgeRoutingKey()); + if (optional.isPresent()) { + edge = optional.get(); + tenantId = edge.getTenantId(); + try { + if (edge.getSecret().equals(request.getEdgeSecret())) { + sessionOpenListener.accept(edge.getId(), this); + edgeVersion = request.getEdgeVersion(); + processSaveEdgeVersionAsAttribute(request.getEdgeVersion().name()); + return ConnectResponseMsg.newBuilder() + .setResponseCode(ConnectResponseCode.ACCEPTED) + .setErrorMsg("") + .setConfiguration(ctx.getEdgeMsgConstructor().constructEdgeConfiguration(edge)) + .setMaxInboundMessageSize(maxInboundMessageSize) + .build(); + } + String error = "Failed to validate the edge!"; + String failureMsg = String.format("%s Provided request secret: %s", error, request.getEdgeSecret()); + ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) + .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(error).build()); + return ConnectResponseMsg.newBuilder() + .setResponseCode(ConnectResponseCode.BAD_CREDENTIALS) + .setErrorMsg(failureMsg) + .setConfiguration(EdgeConfiguration.getDefaultInstance()).build(); + } catch (Exception e) { + String failureMsg = "Failed to process edge connection!"; + ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) + .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(e.getMessage()).build()); + log.error(failureMsg, e); + return ConnectResponseMsg.newBuilder() + .setResponseCode(ConnectResponseCode.SERVER_UNAVAILABLE) + .setErrorMsg(failureMsg) + .setConfiguration(EdgeConfiguration.getDefaultInstance()).build(); + } + } + return ConnectResponseMsg.newBuilder() + .setResponseCode(ConnectResponseCode.BAD_CREDENTIALS) + .setErrorMsg("Failed to find the edge! Routing key: " + request.getEdgeRoutingKey()) + .setConfiguration(EdgeConfiguration.getDefaultInstance()).build(); + } + + private void processSaveEdgeVersionAsAttribute(String edgeVersion) { + AttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(new StringDataEntry(DataConstants.EDGE_VERSION_ATTR_KEY, edgeVersion), System.currentTimeMillis()); + ctx.getAttributesService().save(tenantId, edge.getId(), AttributeScope.SERVER_SCOPE, attributeKvEntry); + } + + private void interruptGeneralProcessingOnSync() { + log.debug("[{}][{}][{}] Sync process started. General processing interrupted!", tenantId, edge.getId(), sessionId); + stopCurrentSendDownlinkMsgsTask(true); + } + + protected ListenableFuture sendDownlinkMsgsPack(List downlinkMsgsPack) { + interruptPreviousSendDownlinkMsgsTask(); + + sessionState.setSendDownlinkMsgsFuture(SettableFuture.create()); + sessionState.getPendingMsgsMap().clear(); + + downlinkMsgsPack.forEach(msg -> sessionState.getPendingMsgsMap().put(msg.getDownlinkMsgId(), msg)); + scheduleDownlinkMsgsPackSend(1); + + return sessionState.getSendDownlinkMsgsFuture(); + } + + private void interruptPreviousSendDownlinkMsgsTask() { + if (sessionState.getSendDownlinkMsgsFuture() != null && !sessionState.getSendDownlinkMsgsFuture().isDone() + || sessionState.getScheduledSendDownlinkTask() != null && !sessionState.getScheduledSendDownlinkTask().isCancelled()) { + log.debug("[{}][{}][{}] Previous send downlink future was not properly completed, stopping it now!", tenantId, edge.getId(), sessionId); + stopCurrentSendDownlinkMsgsTask(true); + } + } + + private void onUplinkMsg(UplinkMsg uplinkMsg) { + if (isRateLimitViolated(uplinkMsg)) { + return; + } + ListenableFuture> future = processUplinkMsg(uplinkMsg); + Futures.addCallback(future, new FutureCallback<>() { + @Override + public void onSuccess(@Nullable List result) { + sendResponseMessage(uplinkMsg.getUplinkMsgId(), true, null); + } + + @Override + public void onFailure(Throwable t) { + String errorMsg = EdgeUtils.createErrorMsgFromRootCauseAndStackTrace(t); + sendResponseMessage(uplinkMsg.getUplinkMsgId(), false, errorMsg); + } + }, ctx.getGrpcCallbackExecutorService()); + } + + private boolean isRateLimitViolated(UplinkMsg uplinkMsg) { + if (!ctx.getRateLimitService().checkRateLimit(LimitedApi.EDGE_UPLINK_MESSAGES, tenantId) || + !ctx.getRateLimitService().checkRateLimit(LimitedApi.EDGE_UPLINK_MESSAGES_PER_EDGE, tenantId, edge.getId())) { + String errorMsg = String.format("Failed to process uplink message. %s", RATE_LIMIT_REACHED); + sendResponseMessage(uplinkMsg.getUplinkMsgId(), false, errorMsg); + return true; + } + return false; + } + + private void scheduleDownlinkMsgsPackSend(int attempt) { + Runnable sendDownlinkMsgsTask = () -> { + try { + if (isConnected() && !sessionState.getPendingMsgsMap().values().isEmpty()) { + List copy = new ArrayList<>(sessionState.getPendingMsgsMap().values()); + if (attempt > 1) { + String error = "Failed to deliver the batch"; + String failureMsg = String.format("{%s}: {%s}", error, copy); + if (attempt == 2) { + // Send a failure notification only on the second attempt. + // This ensures that failure alerts are sent just once to avoid redundant notifications. + ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId) + .edgeId(edge.getId()).customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(error).build()); + } + log.warn("[{}][{}] {}, attempt: {}", tenantId, sessionId, failureMsg, attempt); + } + log.trace("[{}][{}][{}] downlink msg(s) are going to be send.", tenantId, sessionId, copy.size()); + for (DownlinkMsg downlinkMsg : copy) { + if (clientMaxInboundMessageSize != 0 && downlinkMsg.getSerializedSize() > clientMaxInboundMessageSize) { + String error = String.format("Client max inbound message size %s is exceeded. Please increase value of CLOUD_RPC_MAX_INBOUND_MESSAGE_SIZE " + + "env variable on the edge and restart it.", clientMaxInboundMessageSize); + String message = String.format("Downlink msg size %s exceeds client max inbound message size %s. " + + "Please increase value of CLOUD_RPC_MAX_INBOUND_MESSAGE_SIZE env variable on the edge and restart it.", downlinkMsg.getSerializedSize(), clientMaxInboundMessageSize); + log.error("[{}][{}][{}] {} Message {}", tenantId, edge.getId(), sessionId, message, downlinkMsg); + ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId) + .edgeId(edge.getId()).customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(message).error(error).build()); + sessionState.getPendingMsgsMap().remove(downlinkMsg.getDownlinkMsgId()); + } else { + sendDownlinkMsg(ResponseMsg.newBuilder() + .setDownlinkMsg(downlinkMsg) + .build()); + } + } + if (attempt < MAX_DOWNLINK_ATTEMPTS) { + scheduleDownlinkMsgsPackSend(attempt + 1); + } else { + String failureMsg = String.format("Failed to deliver messages: %s", copy); + log.warn("[{}][{}] Failed to deliver the batch after {} attempts. Next messages are going to be discarded {}", + tenantId, sessionId, MAX_DOWNLINK_ATTEMPTS, copy); + ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) + .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg) + .error("Failed to deliver messages after " + MAX_DOWNLINK_ATTEMPTS + " attempts").build()); + stopCurrentSendDownlinkMsgsTask(false); + } + } else { + stopCurrentSendDownlinkMsgsTask(false); + } + } catch (Exception e) { + log.warn("[{}][{}] Failed to send downlink msgs. Error msg {}", tenantId, sessionId, e.getMessage(), e); + stopCurrentSendDownlinkMsgsTask(true); + } + }; + + if (attempt == 1) { + sendDownlinkExecutorService.submit(sendDownlinkMsgsTask); + } else { + sessionState.setScheduledSendDownlinkTask( + sendDownlinkExecutorService.schedule( + sendDownlinkMsgsTask, + ctx.getEdgeEventStorageSettings().getSleepIntervalBetweenBatches(), + TimeUnit.MILLISECONDS) + ); + } + } + + private void sendResponseMessage(int uplinkMsgId, boolean success, String errorMsg) { + UplinkResponseMsg.Builder responseBuilder = UplinkResponseMsg.newBuilder() + .setUplinkMsgId(uplinkMsgId) + .setSuccess(success); + if (errorMsg != null) { + responseBuilder.setErrorMsg(errorMsg); + } + sendDownlinkMsg(ResponseMsg.newBuilder() + .setUplinkResponseMsg(responseBuilder.build()) + .build()); + } + + private void onDownlinkResponse(DownlinkResponseMsg msg) { + try { + if (msg.getSuccess()) { + sessionState.getPendingMsgsMap().remove(msg.getDownlinkMsgId()); + log.debug("[{}][{}] Msg has been processed successfully! Msg Id: [{}], Msg: {}", tenantId, edge.getRoutingKey(), msg.getDownlinkMsgId(), msg); + } else { + log.error("[{}][{}] Msg processing failed! Msg Id: [{}], Error msg: {}", tenantId, edge.getRoutingKey(), msg.getDownlinkMsgId(), msg.getErrorMsg()); + } + if (sessionState.getPendingMsgsMap().isEmpty()) { + log.debug("[{}][{}] Pending msgs map is empty. Stopping current iteration", tenantId, edge.getRoutingKey()); + stopCurrentSendDownlinkMsgsTask(false); + } + } catch (Exception e) { + log.error("[{}][{}] Can't process downlink response message [{}]", tenantId, sessionId, msg, e); + } + } + + public void processHighPriorityEvents() { + try { + List highPriorityEvents = new ArrayList<>(); + EdgeEvent event; + while ((event = highPriorityQueue.poll()) != null) { + highPriorityEvents.add(event); + } + List downlinkMsgsPack = convertToDownlinkMsgsPack(highPriorityEvents); + sendDownlinkMsgsPack(downlinkMsgsPack).get(); + } catch (Exception e) { + log.error("[{}] Failed to process high priority events", sessionId, e); + } + } + + public ListenableFuture processEdgeEvents() throws Exception { + SettableFuture result = SettableFuture.create(); + log.trace("[{}][{}] starting processing edge events", tenantId, sessionId); + if (isConnected() && isSyncCompleted()) { + Pair startTsAndSeqId = getQueueStartTsAndSeqId().get(); + previousStartTs = startTsAndSeqId.getFirst(); + previousStartSeqId = startTsAndSeqId.getSecond(); + GeneralEdgeEventFetcher fetcher = new GeneralEdgeEventFetcher( + previousStartTs, + previousStartSeqId, + seqIdEnd, + false, + Integer.toUnsignedLong(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount()), + ctx.getEdgeEventService()); + Futures.addCallback(startProcessingEdgeEvents(fetcher), new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Pair newStartTsAndSeqId) { + if (newStartTsAndSeqId != null) { + ListenableFuture> updateFuture = updateQueueStartTsAndSeqId(newStartTsAndSeqId); + Futures.addCallback(updateFuture, new FutureCallback<>() { + @Override + public void onSuccess(@Nullable List list) { + log.debug("[{}][{}] queue offset was updated [{}]", tenantId, sessionId, newStartTsAndSeqId); + if (fetcher.isSeqIdNewCycleStarted()) { + seqIdEnd = fetcher.getSeqIdEnd(); + boolean newEventsAvailable = isNewEdgeEventsAvailable(); + result.set(newEventsAvailable); + } else { + seqIdEnd = null; + boolean newEventsAvailable = isSeqIdStartedNewCycle(); + if (!newEventsAvailable) { + newEventsAvailable = isNewEdgeEventsAvailable(); + } + result.set(newEventsAvailable); + } + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}][{}] Failed to update queue offset [{}]", tenantId, sessionId, newStartTsAndSeqId, t); + result.setException(t); + } + }, ctx.getGrpcCallbackExecutorService()); + } else { + log.trace("[{}][{}] newStartTsAndSeqId is null. Skipping iteration without db update", tenantId, sessionId); + result.set(Boolean.FALSE); + } + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}][{}] Failed to process events", tenantId, sessionId, t); + result.setException(t); + } + }, ctx.getGrpcCallbackExecutorService()); + } else { + log.trace("[{}][{}] edge is not connected or sync is not completed. Skipping iteration", tenantId, sessionId); + result.set(null); + } + return result; + } + + protected List convertToDownlinkMsgsPack(List edgeEvents) { + List result = new ArrayList<>(); + for (EdgeEvent edgeEvent : edgeEvents) { + log.trace("[{}][{}] converting edge event to downlink msg [{}]", tenantId, sessionId, edgeEvent); + DownlinkMsg downlinkMsg = null; + try { + switch (edgeEvent.getAction()) { + case UPDATED, ADDED, DELETED, ASSIGNED_TO_EDGE, UNASSIGNED_FROM_EDGE, ALARM_ACK, ALARM_CLEAR, + ALARM_DELETE, CREDENTIALS_UPDATED, RELATION_ADD_OR_UPDATE, RELATION_DELETED, RPC_CALL, + ASSIGNED_TO_CUSTOMER, UNASSIGNED_FROM_CUSTOMER, ADDED_COMMENT, UPDATED_COMMENT, DELETED_COMMENT -> { + downlinkMsg = convertEntityEventToDownlink(edgeEvent); + if (downlinkMsg != null && downlinkMsg.getWidgetTypeUpdateMsgCount() > 0) { + log.trace("[{}][{}] widgetTypeUpdateMsg message processed, downlinkMsgId = {}", tenantId, sessionId, downlinkMsg.getDownlinkMsgId()); + } else { + log.trace("[{}][{}] entity message processed [{}]", tenantId, sessionId, downlinkMsg); + } + } + case ATTRIBUTES_UPDATED, POST_ATTRIBUTES, ATTRIBUTES_DELETED, TIMESERIES_UPDATED -> + downlinkMsg = ctx.getTelemetryProcessor().convertTelemetryEventToDownlink(edge, edgeEvent); + default -> log.warn("[{}][{}] Unsupported action type [{}]", tenantId, sessionId, edgeEvent.getAction()); + } + } catch (Exception e) { + log.error("[{}][{}] Exception during converting edge event to downlink msg", tenantId, sessionId, e); + } + if (downlinkMsg != null) { + result.add(downlinkMsg); + } + } + return result; + } + + private ListenableFuture> getQueueStartTsAndSeqId() { + ListenableFuture> future = + ctx.getAttributesService().find(edge.getTenantId(), edge.getId(), AttributeScope.SERVER_SCOPE, Arrays.asList(QUEUE_START_TS_ATTR_KEY, QUEUE_START_SEQ_ID_ATTR_KEY)); + return Futures.transform(future, attributeKvEntries -> { + long startTs = 0L; + long startSeqId = 0L; + for (AttributeKvEntry attributeKvEntry : attributeKvEntries) { + if (QUEUE_START_TS_ATTR_KEY.equals(attributeKvEntry.getKey())) { + startTs = attributeKvEntry.getLongValue().isPresent() ? attributeKvEntry.getLongValue().get() : 0L; + } + if (QUEUE_START_SEQ_ID_ATTR_KEY.equals(attributeKvEntry.getKey())) { + startSeqId = attributeKvEntry.getLongValue().isPresent() ? attributeKvEntry.getLongValue().get() : 0L; + } + } + if (startSeqId == 0L) { + startSeqId = findStartSeqIdFromOldestEventIfAny(); + } + return Pair.of(startTs, startSeqId); + }, ctx.getGrpcCallbackExecutorService()); + } + + private boolean isSeqIdStartedNewCycle() { + try { + TimePageLink pageLink = new TimePageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount(), 0, null, null, newStartTs, System.currentTimeMillis()); + PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), 0L, previousStartSeqId == 0 ? null : previousStartSeqId - 1, pageLink); + return !edgeEvents.getData().isEmpty(); + } catch (Exception e) { + log.error("[{}][{}][{}] Failed to execute isSeqIdStartedNewCycle", tenantId, edge.getId(), sessionId, e); + } + return false; + } + + private boolean isNewEdgeEventsAvailable() { + try { + TimePageLink pageLink = new TimePageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount(), 0, null, null, newStartTs, System.currentTimeMillis()); + PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), newStartSeqId, null, pageLink); + return !edgeEvents.getData().isEmpty() || !highPriorityQueue.isEmpty(); + } catch (Exception e) { + log.error("[{}][{}][{}] Failed to execute isNewEdgeEventsAvailable", tenantId, edge.getId(), sessionId, e); + } + return false; + } + + private long findStartSeqIdFromOldestEventIfAny() { + long startSeqId = 0L; + try { + TimePageLink pageLink = new TimePageLink(1, 0, null, new SortOrder("createdTime"), null, null); + PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), null, null, pageLink); + if (!edgeEvents.getData().isEmpty()) { + startSeqId = edgeEvents.getData().get(0).getSeqId() - 1; + } + } catch (Exception e) { + log.error("[{}][{}][{}] Failed to execute findStartSeqIdFromOldestEventIfAny", tenantId, edge.getId(), sessionId, e); + } + return startSeqId; + } + + private ListenableFuture> updateQueueStartTsAndSeqId(Pair pair) { + newStartTs = pair.getFirst(); + newStartSeqId = pair.getSecond(); + log.trace("[{}] updateQueueStartTsAndSeqId [{}][{}][{}]", sessionId, edge.getId(), newStartTs, newStartSeqId); + List attributes = Arrays.asList( + new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_TS_ATTR_KEY, newStartTs), System.currentTimeMillis()), + new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_SEQ_ID_ATTR_KEY, newStartSeqId), System.currentTimeMillis())); + return ctx.getAttributesService().save(edge.getTenantId(), edge.getId(), AttributeScope.SERVER_SCOPE, attributes); + } + + protected ListenableFuture> startProcessingEdgeEvents(EdgeEventFetcher fetcher) { + SettableFuture> result = SettableFuture.create(); + PageLink pageLink = fetcher.getPageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount()); + processEdgeEvents(fetcher, pageLink, result); + return result; + } + + private void markSyncCompletedSendEdgeEventUpdate() { + syncCompleted = true; + ctx.getClusterService().onEdgeEventUpdate(new EdgeEventUpdateMsg(edge.getTenantId(), edge.getId())); + } + + private void stopCurrentSendDownlinkMsgsTask(Boolean isInterrupted) { + if (sessionState.getSendDownlinkMsgsFuture() != null && !sessionState.getSendDownlinkMsgsFuture().isDone()) { + sessionState.getSendDownlinkMsgsFuture().set(isInterrupted); + } + if (sessionState.getScheduledSendDownlinkTask() != null) { + sessionState.getScheduledSendDownlinkTask().cancel(true); + } + } + + private void sendDownlinkMsg(ResponseMsg downlinkMsg) { + if (downlinkMsg.getDownlinkMsg().getWidgetTypeUpdateMsgCount() > 0) { + log.trace("[{}][{}] Sending downlink widgetTypeUpdateMsg, downlinkMsgId = {}", tenantId, sessionId, downlinkMsg.getDownlinkMsg().getDownlinkMsgId()); + } else { + log.trace("[{}][{}] Sending downlink msg [{}]", tenantId, sessionId, downlinkMsg); + } + if (isConnected()) { + downlinkMsgLock.lock(); + try { + outputStream.onNext(downlinkMsg); + } catch (Exception e) { + log.error("[{}][{}] Failed to send downlink message [{}]", tenantId, sessionId, downlinkMsg, e); + connected = false; + sessionCloseListener.accept(edge, sessionId); + } finally { + downlinkMsgLock.unlock(); + } + log.trace("[{}][{}] Response msg successfully sent. downlinkMsgId = {}", tenantId, sessionId, downlinkMsg.getDownlinkMsg().getDownlinkMsgId()); + } + } + + protected DownlinkMsg convertEntityEventToDownlink(EdgeEvent edgeEvent) { + log.trace("[{}] Executing convertEntityEventToDownlink, edgeEvent [{}], action [{}]", edgeEvent.getTenantId(), edgeEvent, edgeEvent.getAction()); + return switch (edgeEvent.getType()) { + case EDGE -> ctx.getEdgeProcessor().convertEdgeEventToDownlink(edgeEvent); + case DEVICE -> ctx.getDeviceProcessor().convertDeviceEventToDownlink(edgeEvent, edgeVersion); + case DEVICE_PROFILE -> ctx.getDeviceProfileProcessor().convertDeviceProfileEventToDownlink(edgeEvent, edgeVersion); + case ASSET_PROFILE -> ctx.getAssetProfileProcessor().convertAssetProfileEventToDownlink(edgeEvent, edgeVersion); + case ASSET -> ctx.getAssetProcessor().convertAssetEventToDownlink(edgeEvent, edgeVersion); + case ENTITY_VIEW -> ctx.getEntityViewProcessor().convertEntityViewEventToDownlink(edgeEvent, edgeVersion); + case DASHBOARD -> ctx.getDashboardProcessor().convertDashboardEventToDownlink(edgeEvent, edgeVersion); + case CUSTOMER -> ctx.getCustomerProcessor().convertCustomerEventToDownlink(edgeEvent, edgeVersion); + case RULE_CHAIN -> ctx.getRuleChainProcessor().convertRuleChainEventToDownlink(edgeEvent, edgeVersion); + case RULE_CHAIN_METADATA -> ctx.getRuleChainProcessor().convertRuleChainMetadataEventToDownlink(edgeEvent, edgeVersion); + case ALARM -> ctx.getAlarmProcessor().convertAlarmEventToDownlink(edgeEvent, edgeVersion); + case ALARM_COMMENT -> ctx.getAlarmProcessor().convertAlarmCommentEventToDownlink(edgeEvent, edgeVersion); + case USER -> ctx.getUserProcessor().convertUserEventToDownlink(edgeEvent, edgeVersion); + case RELATION -> ctx.getRelationProcessor().convertRelationEventToDownlink(edgeEvent, edgeVersion); + case WIDGETS_BUNDLE -> ctx.getWidgetBundleProcessor().convertWidgetsBundleEventToDownlink(edgeEvent, edgeVersion); + case WIDGET_TYPE -> ctx.getWidgetTypeProcessor().convertWidgetTypeEventToDownlink(edgeEvent, edgeVersion); + case ADMIN_SETTINGS -> ctx.getAdminSettingsProcessor().convertAdminSettingsEventToDownlink(edgeEvent, edgeVersion); + case OTA_PACKAGE -> ctx.getOtaPackageProcessor().convertOtaPackageEventToDownlink(edgeEvent, edgeVersion); + case TB_RESOURCE -> ctx.getResourceProcessor().convertResourceEventToDownlink(edgeEvent, edgeVersion); + case QUEUE -> ctx.getQueueProcessor().convertQueueEventToDownlink(edgeEvent, edgeVersion); + case TENANT -> ctx.getTenantProcessor().convertTenantEventToDownlink(edgeEvent, edgeVersion); + case TENANT_PROFILE -> ctx.getTenantProfileProcessor().convertTenantProfileEventToDownlink(edgeEvent, edgeVersion); + case NOTIFICATION_RULE -> ctx.getNotificationEdgeProcessor().convertNotificationRuleToDownlink(edgeEvent); + case NOTIFICATION_TARGET -> ctx.getNotificationEdgeProcessor().convertNotificationTargetToDownlink(edgeEvent); + case NOTIFICATION_TEMPLATE -> ctx.getNotificationEdgeProcessor().convertNotificationTemplateToDownlink(edgeEvent); + case OAUTH2_CLIENT -> ctx.getOAuth2EdgeProcessor().convertOAuth2ClientEventToDownlink(edgeEvent, edgeVersion); + case DOMAIN -> ctx.getOAuth2EdgeProcessor().convertOAuth2DomainEventToDownlink(edgeEvent, edgeVersion); + default -> { + log.warn("[{}] Unsupported edge event type [{}]", edgeEvent.getTenantId(), edgeEvent); + yield null; + } + }; + } + + public void addEventToHighPriorityQueue(EdgeEvent edgeEvent) { + while (highPriorityQueue.size() > maxHighPriorityQueueSizePerSession) { + EdgeEvent oldestHighPriority = highPriorityQueue.poll(); + if (oldestHighPriority != null) { + log.warn("[{}][{}][{}] High priority queue is full. Removing oldest high priority event from queue {}", + tenantId, edge.getId(), sessionId, oldestHighPriority); + } + } + highPriorityQueue.add(edgeEvent); + } + + protected ListenableFuture> processUplinkMsg(UplinkMsg uplinkMsg) { + List> result = new ArrayList<>(); + try { + if (uplinkMsg.getDeviceProfileUpdateMsgCount() > 0) { + for (DeviceProfileUpdateMsg deviceProfileUpdateMsg : uplinkMsg.getDeviceProfileUpdateMsgList()) { + result.add(((DeviceProfileProcessor) ctx.getDeviceProfileEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processDeviceProfileMsgFromEdge(edge.getTenantId(), edge, deviceProfileUpdateMsg)); + } + } + if (uplinkMsg.getDeviceUpdateMsgCount() > 0) { + for (DeviceUpdateMsg deviceUpdateMsg : uplinkMsg.getDeviceUpdateMsgList()) { + result.add(((DeviceProcessor) ctx.getDeviceEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processDeviceMsgFromEdge(edge.getTenantId(), edge, deviceUpdateMsg)); + } + } + if (uplinkMsg.getDeviceCredentialsUpdateMsgCount() > 0) { + for (DeviceCredentialsUpdateMsg deviceCredentialsUpdateMsg : uplinkMsg.getDeviceCredentialsUpdateMsgList()) { + result.add(((DeviceProcessor) ctx.getDeviceEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processDeviceCredentialsMsgFromEdge(edge.getTenantId(), edge.getId(), deviceCredentialsUpdateMsg)); + } + } + if (uplinkMsg.getAssetProfileUpdateMsgCount() > 0) { + for (AssetProfileUpdateMsg assetProfileUpdateMsg : uplinkMsg.getAssetProfileUpdateMsgList()) { + result.add(((AssetProfileProcessor) ctx.getAssetProfileEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processAssetProfileMsgFromEdge(edge.getTenantId(), edge, assetProfileUpdateMsg)); + } + } + if (uplinkMsg.getAssetUpdateMsgCount() > 0) { + for (AssetUpdateMsg assetUpdateMsg : uplinkMsg.getAssetUpdateMsgList()) { + result.add(((AssetProcessor) ctx.getAssetEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processAssetMsgFromEdge(edge.getTenantId(), edge, assetUpdateMsg)); + } + } + if (uplinkMsg.getEntityViewUpdateMsgCount() > 0) { + for (EntityViewUpdateMsg entityViewUpdateMsg : uplinkMsg.getEntityViewUpdateMsgList()) { + result.add(((EntityViewProcessor) ctx.getEntityViewProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processEntityViewMsgFromEdge(edge.getTenantId(), edge, entityViewUpdateMsg)); + } + } + if (uplinkMsg.getEntityDataCount() > 0) { + for (EntityDataProto entityData : uplinkMsg.getEntityDataList()) { + result.addAll(ctx.getTelemetryProcessor().processTelemetryMsg(edge.getTenantId(), entityData)); + } + } + if (uplinkMsg.getAlarmUpdateMsgCount() > 0) { + for (AlarmUpdateMsg alarmUpdateMsg : uplinkMsg.getAlarmUpdateMsgList()) { + result.add(((AlarmProcessor) ctx.getAlarmEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processAlarmMsgFromEdge(edge.getTenantId(), edge.getId(), alarmUpdateMsg)); + } + } + if (uplinkMsg.getAlarmCommentUpdateMsgCount() > 0) { + for (AlarmCommentUpdateMsg alarmCommentUpdateMsg : uplinkMsg.getAlarmCommentUpdateMsgList()) { + result.add(((AlarmProcessor) ctx.getAlarmEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processAlarmCommentMsgFromEdge(edge.getTenantId(), edge.getId(), alarmCommentUpdateMsg)); + } + } + if (uplinkMsg.getRelationUpdateMsgCount() > 0) { + for (RelationUpdateMsg relationUpdateMsg : uplinkMsg.getRelationUpdateMsgList()) { + result.add(((RelationProcessor) ctx.getRelationEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processRelationMsgFromEdge(edge.getTenantId(), edge, relationUpdateMsg)); + } + } + if (uplinkMsg.getDashboardUpdateMsgCount() > 0) { + for (DashboardUpdateMsg dashboardUpdateMsg : uplinkMsg.getDashboardUpdateMsgList()) { + result.add(((DashboardProcessor) ctx.getDashboardEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processDashboardMsgFromEdge(edge.getTenantId(), edge, dashboardUpdateMsg)); + } + } + if (uplinkMsg.getResourceUpdateMsgCount() > 0) { + for (ResourceUpdateMsg resourceUpdateMsg : uplinkMsg.getResourceUpdateMsgList()) { + result.add(((ResourceProcessor) ctx.getResourceEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processResourceMsgFromEdge(edge.getTenantId(), edge, resourceUpdateMsg)); + } + } + if (uplinkMsg.getRuleChainMetadataRequestMsgCount() > 0) { + for (RuleChainMetadataRequestMsg ruleChainMetadataRequestMsg : uplinkMsg.getRuleChainMetadataRequestMsgList()) { + result.add(ctx.getEdgeRequestsService().processRuleChainMetadataRequestMsg(edge.getTenantId(), edge, ruleChainMetadataRequestMsg)); + } + } + if (uplinkMsg.getAttributesRequestMsgCount() > 0) { + for (AttributesRequestMsg attributesRequestMsg : uplinkMsg.getAttributesRequestMsgList()) { + result.add(ctx.getEdgeRequestsService().processAttributesRequestMsg(edge.getTenantId(), edge, attributesRequestMsg)); + } + } + if (uplinkMsg.getRelationRequestMsgCount() > 0) { + for (RelationRequestMsg relationRequestMsg : uplinkMsg.getRelationRequestMsgList()) { + result.add(ctx.getEdgeRequestsService().processRelationRequestMsg(edge.getTenantId(), edge, relationRequestMsg)); + } + } + if (uplinkMsg.getUserCredentialsRequestMsgCount() > 0) { + for (UserCredentialsRequestMsg userCredentialsRequestMsg : uplinkMsg.getUserCredentialsRequestMsgList()) { + result.add(ctx.getEdgeRequestsService().processUserCredentialsRequestMsg(edge.getTenantId(), edge, userCredentialsRequestMsg)); + } + } + if (uplinkMsg.getDeviceCredentialsRequestMsgCount() > 0) { + for (DeviceCredentialsRequestMsg deviceCredentialsRequestMsg : uplinkMsg.getDeviceCredentialsRequestMsgList()) { + result.add(ctx.getEdgeRequestsService().processDeviceCredentialsRequestMsg(edge.getTenantId(), edge, deviceCredentialsRequestMsg)); + } + } + if (uplinkMsg.getDeviceRpcCallMsgCount() > 0) { + for (DeviceRpcCallMsg deviceRpcCallMsg : uplinkMsg.getDeviceRpcCallMsgList()) { + result.add(((DeviceProcessor) ctx.getDeviceEdgeProcessorFactory().getProcessorByEdgeVersion(edgeVersion)) + .processDeviceRpcCallFromEdge(edge.getTenantId(), edge, deviceRpcCallMsg)); + } + } + if (uplinkMsg.getWidgetBundleTypesRequestMsgCount() > 0) { + for (WidgetBundleTypesRequestMsg widgetBundleTypesRequestMsg : uplinkMsg.getWidgetBundleTypesRequestMsgList()) { + result.add(ctx.getEdgeRequestsService().processWidgetBundleTypesRequestMsg(edge.getTenantId(), edge, widgetBundleTypesRequestMsg)); + } + } + if (uplinkMsg.getEntityViewsRequestMsgCount() > 0) { + for (EntityViewsRequestMsg entityViewRequestMsg : uplinkMsg.getEntityViewsRequestMsgList()) { + result.add(ctx.getEdgeRequestsService().processEntityViewsRequestMsg(edge.getTenantId(), edge, entityViewRequestMsg)); + } + } + } catch (Exception e) { + String failureMsg = String.format("Can't process uplink msg [%s] from edge", uplinkMsg); + log.error("[{}][{}] Can't process uplink msg [{}]", edge.getTenantId(), sessionId, uplinkMsg, e); + ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(edge.getTenantId()).edgeId(edge.getId()) + .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(e.getMessage()).build()); + return Futures.immediateFailedFuture(e); + } + return Futures.allAsList(result); + } + + protected void destroy() { + // used for KafkaEdgeGrpcSession only + } + + protected void deleteTopic(EdgeId edgeId) { + // used for KafkaEdgeGrpcSession only + } + + @Override + public void close() { + log.debug("[{}][{}] Closing session", tenantId, sessionId); + connected = false; + try { + outputStream.onCompleted(); + } catch (Exception e) { + log.debug("[{}][{}] Failed to close output stream: {}", tenantId, sessionId, e.getMessage()); + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java index a65e6ab85f..6aaa3d72a4 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java @@ -30,6 +30,10 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificat import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; +import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.service.edge.EdgeContextComponent; @@ -42,22 +46,29 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.function.BiConsumer; @Slf4j -public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession { +public class KafkaEdgeGrpcSession extends EdgeGrpcSession { + private final TopicService topicService; private final TbCoreQueueFactory tbCoreQueueFactory; + private final TbKafkaSettings kafkaSettings; + private final TbKafkaTopicConfigs kafkaTopicConfigs; + private volatile boolean isHighPriorityProcessing; private QueueConsumerManager> consumer; private ExecutorService consumerExecutor; - public KafkaEdgeGrpcSession(EdgeContextComponent ctx, TbCoreQueueFactory tbCoreQueueFactory, StreamObserver outputStream, - BiConsumer sessionOpenListener, - BiConsumer sessionCloseListener, ScheduledExecutorService sendDownlinkExecutorService, - int maxInboundMessageSize, int maxHighPriorityQueueSizePerSession) { + public KafkaEdgeGrpcSession(EdgeContextComponent ctx, TopicService topicService, TbCoreQueueFactory tbCoreQueueFactory, + TbKafkaSettings kafkaSettings, TbKafkaTopicConfigs kafkaTopicConfigs, StreamObserver outputStream, + BiConsumer sessionOpenListener, BiConsumer sessionCloseListener, + ScheduledExecutorService sendDownlinkExecutorService, int maxInboundMessageSize, int maxHighPriorityQueueSizePerSession) { super(ctx, outputStream, sessionOpenListener, sessionCloseListener, sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession); + this.topicService = topicService; this.tbCoreQueueFactory = tbCoreQueueFactory; + this.kafkaSettings = kafkaSettings; + this.kafkaTopicConfigs = kafkaTopicConfigs; } private void processMsgs(List> msgs, TbQueueConsumer> consumer) { @@ -125,4 +136,11 @@ public class KafkaEdgeGrpcSession extends AbstractEdgeGrpcSession { consumerExecutor.shutdown(); } + @Override + public void deleteTopic(EdgeId edgeId) { + String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); + TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + kafkaAdmin.deleteTopic(topic); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java index cddd895c02..12516568a8 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/PostgresEdgeGrpcSession.java @@ -29,19 +29,16 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.function.BiConsumer; @Slf4j -public class PostgresEdgeGrpcSession extends AbstractEdgeGrpcSession { +public class PostgresEdgeGrpcSession extends EdgeGrpcSession { PostgresEdgeGrpcSession(EdgeContextComponent ctx, StreamObserver outputStream, - BiConsumer sessionOpenListener, + BiConsumer sessionOpenListener, BiConsumer sessionCloseListener, ScheduledExecutorService sendDownlinkExecutorService, int maxInboundMessageSize, int maxHighPriorityQueueSizePerSession) { super(ctx, outputStream, sessionOpenListener, sessionCloseListener, sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession); initInputStream(); } - @Override - public void destroy() {} - @Override public ListenableFuture migrateEdgeEvents() { return Futures.immediateFuture(Boolean.FALSE); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 38cebbb509..bce599c0e3 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -35,7 +35,6 @@ import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; @@ -55,12 +54,7 @@ import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.kafka.TbKafkaAdmin; -import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; -import java.util.Optional; import java.util.Set; @Slf4j @@ -68,12 +62,8 @@ import java.util.Set; @RequiredArgsConstructor public class EntityStateSourcingListener { - private final TopicService topicService; - private final TbClusterService tbClusterService; private final TenantService tenantService; - - private final Optional kafkaSettings; - private final Optional kafkaTopicConfigs; + private final TbClusterService tbClusterService; @PostConstruct public void init() { @@ -147,7 +137,7 @@ public class EntityStateSourcingListener { log.debug("[{}][{}][{}] Handling entity deletion event: {}", tenantId, entityType, entityId, event); switch (entityType) { - case ASSET, ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, NOTIFICATION_RULE -> { + case ASSET, ASSET_PROFILE, EDGE, ENTITY_VIEW, CUSTOMER, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } case NOTIFICATION_REQUEST -> { @@ -187,10 +177,6 @@ public class EntityStateSourcingListener { TbResourceInfo tbResource = (TbResourceInfo) event.getEntity(); tbClusterService.onResourceDeleted(tbResource, null); } - case EDGE -> { - onEdgeDelete(tenantId, (EdgeId) event.getEntityId()); - tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); - } default -> {} } } @@ -261,14 +247,6 @@ public class EntityStateSourcingListener { } } - private void onEdgeDelete(TenantId tenantId, EdgeId edgeId) { - if (kafkaSettings.isPresent() && kafkaTopicConfigs.isPresent()) { - String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); - TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings.get(), kafkaTopicConfigs.get().getEdgeEventConfigs()); - kafkaAdmin.deleteTopic(topic); - } - } - private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { String data = JacksonUtil.toString(JacksonUtil.valueToTree(assignedDevice)); if (data != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java index dfd6226ed3..1bd0e30c77 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java @@ -65,20 +65,16 @@ public class KafkaEdgeTopicsCleanUpService { @Value("${sql.ttl.edge_events.edge_events_ttl:2628000}") private long ttlSeconds; - private final ExecutorService executorService = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("kafka-edge-topic-cleanup")); - @Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.edge_events.execution_interval_ms})}", fixedDelayString = "${sql.ttl.edge_events.execution_interval_ms}") public void cleanUp() { - executorService.submit(() -> { - PageDataIterable tenants = new PageDataIterable<>(tenantService::findTenantsIds, 10_000); - for (TenantId tenantId : tenants) { - try { - cleanUp(tenantId); - } catch (Exception e) { - log.warn("Failed to drop kafka topics for tenant {}", tenantId, e); - } + PageDataIterable tenants = new PageDataIterable<>(tenantService::findTenantsIds, 10_000); + for (TenantId tenantId : tenants) { + try { + cleanUp(tenantId); + } catch (Exception e) { + log.warn("Failed to drop kafka topics for tenant {}", tenantId, e); } - }); + } } private void cleanUp(TenantId tenantId) throws Exception { @@ -91,12 +87,12 @@ public class KafkaEdgeTopicsCleanUpService { long ttlMillis = TimeUnit.SECONDS.toChronoUnit().getDuration().multipliedBy(ttlSeconds).toMillis(); for (EdgeId edgeId : edgeIds) { + TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); attributesService.find(tenantId, edgeId, AttributeScope.SERVER_SCOPE, DefaultDeviceStateService.LAST_CONNECT_TIME).get() .flatMap(AttributeKvEntry::getLongValue) .filter(lastConnectTime -> isTopicExpired(lastConnectTime, ttlMillis, currentTimeMillis)) .ifPresent(lastConnectTime -> { String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); - TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); if (kafkaAdmin.isTopicEmpty(topic)) { kafkaAdmin.deleteTopic(topic); log.info("Removed outdated topic for tenant {} and edge with id {} older than {}", diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index 2540cdeafa..7960cbcf32 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -83,7 +83,7 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider @Override public TbQueueProducer> getTbEdgeMsgProducer() { - throw new RuntimeException("Not Implemented! Should not be used Transport!"); + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } @Override @@ -93,7 +93,7 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider @Override public TbQueueProducer> getTbEdgeEventsMsgProducer() { - throw new RuntimeException("Not Implemented! Should not be used Transport!"); + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } @Override From 936ad3c2b252e3f94edcc7dc371862ba27fb6de1 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Mon, 25 Nov 2024 12:19:47 +0200 Subject: [PATCH 39/71] Refactored due to comments --- .../transport/mqtt/MqttTransportHandler.java | 34 +++++++------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 6d5c0f5d50..e2bc8cd768 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -314,26 +314,8 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } break; case SUBSCRIBE: - MqttSubscribeMessage mqttSubscribeMsg = (MqttSubscribeMessage) msg; - log.trace("[{}] Processing subscription [{}]!", sessionId, mqttSubscribeMsg.variableHeader().messageId()); - List grantedQoSList = new ArrayList<>(); - for (MqttTopicSubscription subscription : mqttSubscribeMsg.payload().topicSubscriptions()) { - String topic = subscription.topicFilter(); - MqttQoS reqQoS = subscription.qualityOfService(); - try { - if (MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC.equals(topic)) { - registerSubQoS(topic, grantedQoSList, reqQoS); - } else { - log.warn("[{}] Failed to subscribe to [{}][{}]", sessionId, topic, reqQoS); - grantedQoSList.add(ReturnCodeResolver.getSubscriptionReturnCode(deviceSessionCtx.getMqttVersion(), MqttReasonCodes.SubAck.TOPIC_FILTER_INVALID)); - break; - } - } catch (Exception e) { - log.warn("[{}] Failed to subscribe to [{}][{}]", sessionId, topic, reqQoS, e); - grantedQoSList.add(ReturnCodeResolver.getSubscriptionReturnCode(deviceSessionCtx.getMqttVersion(), MqttReasonCodes.SubAck.IMPLEMENTATION_SPECIFIC_ERROR)); - } - } - ctx.writeAndFlush(createSubAckMessage(mqttSubscribeMsg.variableHeader().messageId(), grantedQoSList)); + MqttSubscribeMessage subscribeMessage = (MqttSubscribeMessage) msg; + processSubscribe(ctx, subscribeMessage); break; case PINGREQ: ctx.writeAndFlush(new MqttMessage(new MqttFixedHeader(PINGRESP, false, AT_MOST_ONCE, false, 0))); @@ -772,7 +754,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } private void processSubscribe(ChannelHandlerContext ctx, MqttSubscribeMessage mqttMsg) { - if (!checkConnected(ctx, mqttMsg)) { + if (!checkConnected(ctx, mqttMsg) && !deviceSessionCtx.isProvisionOnly()) { ctx.writeAndFlush(createSubAckMessage(mqttMsg.variableHeader().messageId(), Collections.singletonList(MqttReasonCodes.SubAck.NOT_AUTHORIZED.byteValue() & 0xFF))); return; } @@ -782,6 +764,16 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement for (MqttTopicSubscription subscription : mqttMsg.payload().topicSubscriptions()) { String topic = subscription.topicName(); MqttQoS reqQoS = subscription.qualityOfService(); + if (deviceSessionCtx.isProvisionOnly()) { + if (MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC.equals(topic)) { + registerSubQoS(topic, grantedQoSList, reqQoS); + } else { + log.debug("[{}] Failed to subscribe to [{}][{}]", sessionId, topic, reqQoS); + grantedQoSList.add(ReturnCodeResolver.getSubscriptionReturnCode(deviceSessionCtx.getMqttVersion(), MqttReasonCodes.SubAck.TOPIC_FILTER_INVALID)); + } + activityReported = true; + continue; + } if (deviceSessionCtx.isDeviceSubscriptionAttributesTopic(topic)) { processAttributesSubscribe(grantedQoSList, topic, reqQoS, TopicType.V1); activityReported = true; From 44930bf9a1e777e088370cc38903249257a54f23 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Mon, 25 Nov 2024 13:39:13 +0200 Subject: [PATCH 40/71] Added processing for unsubscribe for provisioning only clients --- .../server/transport/mqtt/MqttTransportHandler.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index e2bc8cd768..ed721954a7 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -886,7 +886,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } private void processUnsubscribe(ChannelHandlerContext ctx, MqttUnsubscribeMessage mqttMsg) { - if (!checkConnected(ctx, mqttMsg)) { + if (!checkConnected(ctx, mqttMsg) && !deviceSessionCtx.isProvisionOnly()) { ctx.writeAndFlush(createUnSubAckMessage(mqttMsg.variableHeader().messageId(), Collections.singletonList((short) MqttReasonCodes.UnsubAck.NOT_AUTHORIZED.byteValue()))); return; @@ -900,6 +900,14 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement mqttQoSMap.remove(matcher); try { short resultValue = MqttReasonCodes.UnsubAck.SUCCESS.byteValue(); + if (deviceSessionCtx.isProvisionOnly()) { + if (!matcher.matches(MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC)) { + resultValue = MqttReasonCodes.UnsubAck.TOPIC_FILTER_INVALID.byteValue(); + } + unSubResults.add(resultValue); + activityReported = true; + continue; + } switch (topicName) { case MqttTopics.DEVICE_ATTRIBUTES_TOPIC: case MqttTopics.DEVICE_ATTRIBUTES_SHORT_TOPIC: @@ -930,7 +938,6 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement case MqttTopics.GATEWAY_ATTRIBUTES_TOPIC: case MqttTopics.GATEWAY_RPC_TOPIC: case MqttTopics.GATEWAY_ATTRIBUTES_RESPONSE_TOPIC: - case MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC: case MqttTopics.DEVICE_FIRMWARE_RESPONSES_TOPIC: case MqttTopics.DEVICE_FIRMWARE_ERROR_TOPIC: case MqttTopics.DEVICE_SOFTWARE_RESPONSES_TOPIC: From 52b2c5f9ebe5d3eeb9292dd736c8348f73ec32b4 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Mon, 25 Nov 2024 13:42:57 +0200 Subject: [PATCH 41/71] Added forgotten case --- .../server/transport/mqtt/MqttTransportHandler.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index ed721954a7..54df37c3f6 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -317,6 +317,10 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement MqttSubscribeMessage subscribeMessage = (MqttSubscribeMessage) msg; processSubscribe(ctx, subscribeMessage); break; + case UNSUBSCRIBE: + MqttUnsubscribeMessage unsubscribeMessage = (MqttUnsubscribeMessage) msg; + processUnsubscribe(ctx, unsubscribeMessage); + break; case PINGREQ: ctx.writeAndFlush(new MqttMessage(new MqttFixedHeader(PINGRESP, false, AT_MOST_ONCE, false, 0))); break; From 5c6567eab75a87105e088bb3103e5f2c0a68f37e Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Mon, 25 Nov 2024 18:52:43 +0200 Subject: [PATCH 42/71] UI: Fixed incorrect show mail notification preview --- .../sent-notification-dialog.component.html | 2 +- .../sent-notification-dialog.component.scss | 17 ------------- .../sent/sent-notification-dialog.componet.ts | 24 +++++++++++++++++++ 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/notification/sent/sent-notification-dialog.component.html b/ui-ngx/src/app/modules/home/pages/notification/sent/sent-notification-dialog.component.html index 8e9682b3fd..1fa478d718 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/sent/sent-notification-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/notification/sent/sent-notification-dialog.component.html @@ -213,7 +213,7 @@
{{ preview.processedTemplates.EMAIL.subject }}
-
+
diff --git a/ui-ngx/src/app/modules/home/pages/notification/sent/sent-notification-dialog.component.scss b/ui-ngx/src/app/modules/home/pages/notification/sent/sent-notification-dialog.component.scss index e91221d760..3dd5040587 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/sent/sent-notification-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/pages/notification/sent/sent-notification-dialog.component.scss @@ -179,14 +179,6 @@ letter-spacing: 0.25px; } - .html-content { - margin-top: 12px; - font-weight: 400; - font-size: 14px; - line-height: 20px; - overflow-x: auto; - } - &.mini { font-size: 12px; line-height: 1.25; @@ -233,15 +225,6 @@ } } - .preview-group { - .notification-content { - .html-content, - .html-content * { - all: revert; - } - } - } - .delivery-methods-container { .delivery-method-container { &.interact * { diff --git a/ui-ngx/src/app/modules/home/pages/notification/sent/sent-notification-dialog.componet.ts b/ui-ngx/src/app/modules/home/pages/notification/sent/sent-notification-dialog.componet.ts index 7060baf5cc..c3e6262634 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/sent/sent-notification-dialog.componet.ts +++ b/ui-ngx/src/app/modules/home/pages/notification/sent/sent-notification-dialog.componet.ts @@ -77,6 +77,30 @@ export class SentNotificationDialogComponent extends showRefresh = false; + tinyMceOptions: Record = { + base_url: '/assets/tinymce', + suffix: '.min', + plugins: ['autoresize'], + menubar: false, + toolbar: false, + statusbar: false, + resize: false, + readonly: true, + height: 400, + autofocus: false, + branding: false, + promotion: false, + setup: (ed) => { + ed.on('PreInit', () => { + const document = $(ed.iframeElement.contentDocument); + const body = $('#tinymce', document); + body.attr({contenteditable: false}); + body.css('pointerEvents', 'none'); + body.css('userSelect', 'none'); + }) + } + }; + private authUser: AuthUser = getCurrentAuthUser(this.store); private allowNotificationDeliveryMethods: Array; From 8071795e4d21d8397d02d3cde47555ce907e7774 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Mon, 25 Nov 2024 19:04:08 +0200 Subject: [PATCH 43/71] Fixed wrong example in TBEL round number --- .../assets/help/en_US/rulenode/tbel/generator_node_script_fn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/assets/help/en_US/rulenode/tbel/generator_node_script_fn.md b/ui-ngx/src/assets/help/en_US/rulenode/tbel/generator_node_script_fn.md index 6adfd9f210..da67f02a33 100644 --- a/ui-ngx/src/assets/help/en_US/rulenode/tbel/generator_node_script_fn.md +++ b/ui-ngx/src/assets/help/en_US/rulenode/tbel/generator_node_script_fn.md @@ -41,7 +41,7 @@ All fields in resulting object are mandatory. ```javascript var temperature = 18 + Math.random() * (32 - 18); // Round to at most 2 decimal places (optional) -temperature = Math.round( temperature * 100 ) / 100; +temperature = toFixed(temperature, 2); var msg = { temperature: temperature }; return { msg: msg, metadata: {}, msgType: "POST_TELEMETRY_REQUEST" }; {:copy-code} From 3ddb3300f3e56fc083a483e6bf968bbf66bcecef Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 26 Nov 2024 16:46:03 +0200 Subject: [PATCH 44/71] UI: Add custom translate for description columns --- .../asset-profiles-table-config.resolver.ts | 9 ++++----- .../device-profiles-table-config.resolver.ts | 9 +++++---- .../recipient/recipient-table-config.resolver.ts | 6 ++++-- .../notification/rule/rule-table-config.resolver.ts | 6 ++++-- .../rulechain/rulechains-table-config.resolver.ts | 11 ++++++----- .../tenant-profiles-table-config.resolver.ts | 7 +++++-- 6 files changed, 28 insertions(+), 20 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profiles-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profiles-table-config.resolver.ts index 75463ed871..73226d3ff3 100644 --- a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profiles-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profiles-table-config.resolver.ts @@ -28,13 +28,12 @@ import { DatePipe } from '@angular/common'; import { EntityType, entityTypeResources, entityTypeTranslations } from '@shared/models/entity-type.models'; import { EntityAction } from '@home/models/entity/entity-component.models'; import { DialogService } from '@core/services/dialog.service'; -import { MatDialog } from '@angular/material/dialog'; import { ImportExportService } from '@shared/import-export/import-export.service'; -import { HomeDialogsService } from '@home/dialogs/home-dialogs.service'; import { AssetProfile } from '@shared/models/asset.models'; import { AssetProfileService } from '@core/http/asset-profile.service'; import { AssetProfileComponent } from '@home/components/profile/asset-profile.component'; import { AssetProfileTabsComponent } from './asset-profile-tabs.component'; +import { CustomTranslatePipe } from '@shared/pipe/custom-translate.pipe'; @Injectable() export class AssetProfilesTableConfigResolver { @@ -43,12 +42,11 @@ export class AssetProfilesTableConfigResolver { constructor(private assetProfileService: AssetProfileService, private importExport: ImportExportService, - private homeDialogs: HomeDialogsService, private translate: TranslateService, private datePipe: DatePipe, private dialogService: DialogService, private router: Router, - private dialog: MatDialog) { + private customTranslate: CustomTranslatePipe) { this.config.entityType = EntityType.ASSET_PROFILE; this.config.entityComponent = AssetProfileComponent; @@ -61,7 +59,8 @@ export class AssetProfilesTableConfigResolver { this.config.columns.push( new DateEntityTableColumn('createdTime', 'common.created-time', this.datePipe, '150px'), new EntityTableColumn('name', 'asset-profile.name', '50%'), - new EntityTableColumn('description', 'asset-profile.description', '50%'), + new EntityTableColumn('description', 'asset-profile.description', '50%', + entity => this.customTranslate.transform(entity.description)), new EntityTableColumn('isDefault', 'asset-profile.default', '60px', entity => { return checkBoxCell(entity.default); diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts index 2746dbd5f3..e4f0f7be55 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts @@ -42,7 +42,7 @@ import { AddDeviceProfileDialogData } from '@home/components/profile/add-device-profile-dialog.component'; import { ImportExportService } from '@shared/import-export/import-export.service'; -import { HomeDialogsService } from '@home/dialogs/home-dialogs.service'; +import { CustomTranslatePipe } from '@shared/pipe/custom-translate.pipe'; @Injectable() export class DeviceProfilesTableConfigResolver { @@ -51,12 +51,12 @@ export class DeviceProfilesTableConfigResolver { constructor(private deviceProfileService: DeviceProfileService, private importExport: ImportExportService, - private homeDialogs: HomeDialogsService, private translate: TranslateService, private datePipe: DatePipe, private dialogService: DialogService, private router: Router, - private dialog: MatDialog) { + private dialog: MatDialog, + private customTranslate: CustomTranslatePipe) { this.config.entityType = EntityType.DEVICE_PROFILE; this.config.entityComponent = DeviceProfileComponent; @@ -77,7 +77,8 @@ export class DeviceProfilesTableConfigResolver { new EntityTableColumn('transportType', 'device-profile.transport-type', '20%', (deviceProfile) => { return this.translate.instant(deviceTransportTypeTranslationMap.get(deviceProfile.transportType)); }), - new EntityTableColumn('description', 'device-profile.description', '40%'), + new EntityTableColumn('description', 'device-profile.description', '40%', + entity => this.customTranslate.transform(entity.description)), new EntityTableColumn('isDefault', 'device-profile.default', '60px', entity => { return checkBoxCell(entity.default); diff --git a/ui-ngx/src/app/modules/home/pages/notification/recipient/recipient-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/notification/recipient/recipient-table-config.resolver.ts index 23368b4584..354baf3877 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/recipient/recipient-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/notification/recipient/recipient-table-config.resolver.ts @@ -35,6 +35,7 @@ import { RecipientTableHeaderComponent } from '@home/pages/notification/recipien import { ActivatedRouteSnapshot } from '@angular/router'; import { Injectable } from '@angular/core'; import { DatePipe } from '@angular/common'; +import { CustomTranslatePipe } from '@shared/pipe/custom-translate.pipe'; @Injectable() export class RecipientTableConfigResolver { @@ -44,7 +45,8 @@ export class RecipientTableConfigResolver { constructor(private notificationService: NotificationService, private translate: TranslateService, private dialog: MatDialog, - private datePipe: DatePipe) { + private datePipe: DatePipe, + private customTranslate: CustomTranslatePipe) { this.config.entityType = EntityType.NOTIFICATION_TARGET; this.config.detailsPanelEnabled = false; @@ -82,7 +84,7 @@ export class RecipientTableConfigResolver { (target) => this.translate.instant(NotificationTargetTypeTranslationMap.get(target.configuration.type)), () => ({}), false), new EntityTableColumn('configuration.description', 'notification.description', '60%', - (target) => target.configuration.description || '', + (target) => this.customTranslate.transform(target.configuration.description || ''), () => ({}), false) ); } diff --git a/ui-ngx/src/app/modules/home/pages/notification/rule/rule-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/notification/rule/rule-table-config.resolver.ts index 1a0ec93953..8b23bf09ba 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/rule/rule-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/notification/rule/rule-table-config.resolver.ts @@ -35,6 +35,7 @@ import { import { ActivatedRouteSnapshot } from '@angular/router'; import { Injectable } from '@angular/core'; import { DatePipe } from '@angular/common'; +import { CustomTranslatePipe } from '@shared/pipe/custom-translate.pipe'; @Injectable() export class RuleTableConfigResolver { @@ -44,7 +45,8 @@ export class RuleTableConfigResolver { constructor(private notificationService: NotificationService, private translate: TranslateService, private dialog: MatDialog, - private datePipe: DatePipe) { + private datePipe: DatePipe, + private customTranslate: CustomTranslatePipe) { this.config.entityType = EntityType.NOTIFICATION_RULE; this.config.detailsPanelEnabled = false; @@ -81,7 +83,7 @@ export class RuleTableConfigResolver { (rule) => this.translate.instant(TriggerTypeTranslationMap.get(rule.triggerType)) || '', () => ({}), true), new EntityTableColumn('additionalConfig.description', 'notification.description', '30%', - (target) => target.additionalConfig?.description || '', + (target) => this.customTranslate.transform(target.additionalConfig?.description || ''), () => ({}), false) ); } diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/rulechains-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/rulechain/rulechains-table-config.resolver.ts index 8f3a73f018..46eb1636ac 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/rulechains-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/rulechain/rulechains-table-config.resolver.ts @@ -50,13 +50,12 @@ import { PageLink } from '@shared/models/page/page-link'; import { Edge } from '@shared/models/edge.models'; import { mergeMap } from 'rxjs/operators'; import { PageData } from '@shared/models/page/page-data'; -import { HomeDialogsService } from '@home/dialogs/home-dialogs.service'; +import { CustomTranslatePipe } from '@shared/pipe/custom-translate.pipe'; @Injectable() export class RuleChainsTableConfigResolver { private readonly config: EntityTableConfig = new EntityTableConfig(); - private edge: Edge; constructor(private ruleChainService: RuleChainService, private dialogService: DialogService, @@ -64,10 +63,10 @@ export class RuleChainsTableConfigResolver { private importExport: ImportExportService, private itembuffer: ItemBufferService, private edgeService: EdgeService, - private homeDialogs: HomeDialogsService, private translate: TranslateService, private datePipe: DatePipe, - private router: Router) { + private router: Router, + private customTranslate: CustomTranslatePipe) { this.config.entityType = EntityType.RULE_CHAIN; this.config.entityComponent = RuleChainComponent; this.config.entityTabsComponent = RuleChainTabsComponent; @@ -128,7 +127,9 @@ export class RuleChainsTableConfigResolver { columns.push( new DateEntityTableColumn('createdTime', 'common.created-time', this.datePipe, '150px'), new EntityTableColumn('name', 'rulechain.name', '50%'), - new EntityTableColumn('description', 'rulechain.description', '50%', entity => entity.additionalInfo?.description ?? '') + new EntityTableColumn('description', 'rulechain.description', '50%', + entity => this.customTranslate.transform(entity.additionalInfo?.description || ''), + () => ({}), false) ); if (ruleChainScope === 'tenant' || ruleChainScope === 'edge') { columns.push( diff --git a/ui-ngx/src/app/modules/home/pages/tenant-profile/tenant-profiles-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/tenant-profile/tenant-profiles-table-config.resolver.ts index bd296fbe18..0d0ca87d3e 100644 --- a/ui-ngx/src/app/modules/home/pages/tenant-profile/tenant-profiles-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/tenant-profile/tenant-profiles-table-config.resolver.ts @@ -33,6 +33,7 @@ import { TenantProfileComponent } from '@home/components/profile/tenant-profile. import { TenantProfileTabsComponent } from './tenant-profile-tabs.component'; import { DialogService } from '@core/services/dialog.service'; import { ImportExportService } from '@shared/import-export/import-export.service'; +import { CustomTranslatePipe } from '@shared/pipe/custom-translate.pipe'; @Injectable() export class TenantProfilesTableConfigResolver { @@ -44,7 +45,8 @@ export class TenantProfilesTableConfigResolver { private translate: TranslateService, private datePipe: DatePipe, private router: Router, - private dialogService: DialogService) { + private dialogService: DialogService, + private customTranslate: CustomTranslatePipe) { this.config.entityType = EntityType.TENANT_PROFILE; this.config.entityComponent = TenantProfileComponent; @@ -55,7 +57,8 @@ export class TenantProfilesTableConfigResolver { this.config.columns.push( new DateEntityTableColumn('createdTime', 'common.created-time', this.datePipe, '150px'), new EntityTableColumn('name', 'tenant-profile.name', '40%'), - new EntityTableColumn('description', 'tenant-profile.description', '60%'), + new EntityTableColumn('description', 'tenant-profile.description', '60%', + entity => this.customTranslate.transform(entity.description)), new EntityTableColumn('isDefault', 'tenant-profile.default', '60px', entity => { return checkBoxCell(entity.default); From a589da7f9f6a0103e47357b64da13b45d44c725d Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 26 Nov 2024 17:40:40 +0200 Subject: [PATCH 45/71] UI: Add custom translate for description columns fixed show null --- .../pages/asset-profile/asset-profiles-table-config.resolver.ts | 2 +- .../device-profile/device-profiles-table-config.resolver.ts | 2 +- .../tenant-profile/tenant-profiles-table-config.resolver.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profiles-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profiles-table-config.resolver.ts index 73226d3ff3..f6fa52567b 100644 --- a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profiles-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profiles-table-config.resolver.ts @@ -60,7 +60,7 @@ export class AssetProfilesTableConfigResolver { new DateEntityTableColumn('createdTime', 'common.created-time', this.datePipe, '150px'), new EntityTableColumn('name', 'asset-profile.name', '50%'), new EntityTableColumn('description', 'asset-profile.description', '50%', - entity => this.customTranslate.transform(entity.description)), + entity => this.customTranslate.transform(entity.description || '')), new EntityTableColumn('isDefault', 'asset-profile.default', '60px', entity => { return checkBoxCell(entity.default); diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts index e4f0f7be55..e9125c3fe7 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts @@ -78,7 +78,7 @@ export class DeviceProfilesTableConfigResolver { return this.translate.instant(deviceTransportTypeTranslationMap.get(deviceProfile.transportType)); }), new EntityTableColumn('description', 'device-profile.description', '40%', - entity => this.customTranslate.transform(entity.description)), + entity => this.customTranslate.transform(entity.description || '')), new EntityTableColumn('isDefault', 'device-profile.default', '60px', entity => { return checkBoxCell(entity.default); diff --git a/ui-ngx/src/app/modules/home/pages/tenant-profile/tenant-profiles-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/tenant-profile/tenant-profiles-table-config.resolver.ts index 0d0ca87d3e..78cdccf3b1 100644 --- a/ui-ngx/src/app/modules/home/pages/tenant-profile/tenant-profiles-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/tenant-profile/tenant-profiles-table-config.resolver.ts @@ -58,7 +58,7 @@ export class TenantProfilesTableConfigResolver { new DateEntityTableColumn('createdTime', 'common.created-time', this.datePipe, '150px'), new EntityTableColumn('name', 'tenant-profile.name', '40%'), new EntityTableColumn('description', 'tenant-profile.description', '60%', - entity => this.customTranslate.transform(entity.description)), + entity => this.customTranslate.transform(entity.description || '')), new EntityTableColumn('isDefault', 'tenant-profile.default', '60px', entity => { return checkBoxCell(entity.default); From 907f6a2f73bb4eb38d47aed7eaaa2a44d6bb00d6 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 26 Nov 2024 19:51:35 +0200 Subject: [PATCH 46/71] UI: Fixed highlighting when disabled toggle-selected --- .../pages/admin/mail-server.component.html | 10 +-- .../pages/admin/mail-server.component.scss | 55 -------------- .../inbox/inbox-table-header.component.html | 14 ++-- .../inbox/inbox-table-header.component.scss | 75 ------------------- .../inbox/inbox-table-header.component.ts | 2 +- .../rule-notification-dialog.component.html | 12 ++- .../rule-notification-dialog.component.scss | 54 ------------- .../components/script-lang.component.scss | 7 ++ .../components/toggle-header.component.scss | 4 + 9 files changed, 27 insertions(+), 206 deletions(-) delete mode 100644 ui-ngx/src/app/modules/home/pages/notification/inbox/inbox-table-header.component.scss diff --git a/ui-ngx/src/app/modules/home/pages/admin/mail-server.component.html b/ui-ngx/src/app/modules/home/pages/admin/mail-server.component.html index 14a6358efb..ff10de6fc2 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/mail-server.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/mail-server.component.html @@ -152,12 +152,10 @@ autocomplete="new-username"/>
- - {{ 'admin.oauth2.basic' | translate }} - {{ 'admin.oauth2.oauth2' | translate }} - + + {{ 'admin.oauth2.basic' | translate }} + {{ 'admin.oauth2.oauth2' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/pages/admin/mail-server.component.scss b/ui-ngx/src/app/modules/home/pages/admin/mail-server.component.scss index b87c58bc48..a60be6f5c2 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/mail-server.component.scss +++ b/ui-ngx/src/app/modules/home/pages/admin/mail-server.component.scss @@ -51,60 +51,5 @@ border: 1px solid rgba(0, 0, 0, 0.2); } } - .mat-button-toggle-group.tb-notification-unread-toggle-group { - &.mat-button-toggle-group-appearance-standard { - border: none; - border-radius: 14px; - - .mat-button-toggle + .mat-button-toggle { - border-left: none; - } - } - - .mat-button-toggle { - background: rgba(0, 0, 0, 0.06); - height: 28px; - align-items: center; - display: flex; - - .mat-button-toggle-ripple { - top: 2px; - left: 2px; - right: 2px; - bottom: 2px; - border-radius: 12px; - } - } - - .mat-button-toggle-button { - color: #959595; - } - - .mat-button-toggle-focus-overlay { - border-radius: 14px; - margin: 2px; - } - - .mat-button-toggle-checked .mat-button-toggle-button { - background-color: $tb-primary-color; - color: #fff; - border-radius: 14px; - margin-left: 2px; - margin-right: 2px; - } - - .mat-button-toggle-appearance-standard .mat-button-toggle-label-content { - line-height: 24px; - font-size: 16px; - font-weight: 500; - .mat-pseudo-checkbox { - display: none; - } - } - - .mat-button-toggle-checked.mat-button-toggle-appearance-standard:not(.mat-button-toggle-disabled):hover .mat-button-toggle-focus-overlay { - opacity: .01; - } - } } } diff --git a/ui-ngx/src/app/modules/home/pages/notification/inbox/inbox-table-header.component.html b/ui-ngx/src/app/modules/home/pages/notification/inbox/inbox-table-header.component.html index 7338a32fc2..8e7263914e 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/inbox/inbox-table-header.component.html +++ b/ui-ngx/src/app/modules/home/pages/notification/inbox/inbox-table-header.component.html @@ -16,12 +16,10 @@ -->
- - {{ 'notification.unread' | translate }} - {{ 'notification.all' | translate }} - - + + {{ 'notification.unread' | translate }} + {{ 'notification.all' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/pages/notification/inbox/inbox-table-header.component.scss b/ui-ngx/src/app/modules/home/pages/notification/inbox/inbox-table-header.component.scss deleted file mode 100644 index af69f47306..0000000000 --- a/ui-ngx/src/app/modules/home/pages/notification/inbox/inbox-table-header.component.scss +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -@import "../../../../../../scss/constants"; - -:host ::ng-deep { - .mat-button-toggle-group.tb-notification-unread-toggle-group { - &.mat-button-toggle-group-appearance-standard { - border: none; - border-radius: 14px; - - .mat-button-toggle + .mat-button-toggle { - border-left: none; - } - } - - .mat-button-toggle { - background: rgba(0, 0, 0, 0.06); - height: 28px; - align-items: center; - display: flex; - - .mat-button-toggle-ripple { - top: 2px; - left: 2px; - right: 2px; - bottom: 2px; - border-radius: 14px; - } - } - - .mat-button-toggle-button { - color: #959595; - } - - .mat-button-toggle-focus-overlay { - border-radius: 14px; - margin: 2px; - } - - .mat-button-toggle-checked .mat-button-toggle-button { - background-color: $tb-primary-color; - color: #fff; - border-radius: 14px; - margin-left: 2px; - margin-right: 2px; - } - - .mat-button-toggle-appearance-standard .mat-button-toggle-label-content { - line-height: 24px; - font-size: 14px; - font-weight: 500; - letter-spacing: .25px; - .mat-pseudo-checkbox { - display: none; - } - } - - .mat-button-toggle-checked.mat-button-toggle-appearance-standard:not(.mat-button-toggle-disabled):hover .mat-button-toggle-focus-overlay { - opacity: .01; - } - } -} diff --git a/ui-ngx/src/app/modules/home/pages/notification/inbox/inbox-table-header.component.ts b/ui-ngx/src/app/modules/home/pages/notification/inbox/inbox-table-header.component.ts index f317f452bd..d3f7cf2005 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/inbox/inbox-table-header.component.ts +++ b/ui-ngx/src/app/modules/home/pages/notification/inbox/inbox-table-header.component.ts @@ -23,7 +23,7 @@ import { Notification } from '@shared/models/notification.models'; @Component({ selector: 'tb-inbox-table-header', templateUrl: './inbox-table-header.component.html', - styleUrls: ['inbox-table-header.component.scss'] + styleUrls: [] }) export class InboxTableHeaderComponent extends EntityTableHeaderComponent { diff --git a/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.html b/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.html index 23d76e90a3..2d7ef41bd5 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.html @@ -161,13 +161,11 @@ {{ 'notification.device-activity-trigger-settings' | translate }}
-
- - {{ 'device.device' | translate }} - {{ 'device-profile.device-profile' | translate }} - +
+ + {{ 'device.device' | translate }} + {{ 'device-profile.device-profile' | translate }} +
Date: Wed, 27 Nov 2024 09:19:33 +0200 Subject: [PATCH 47/71] UI: Show SCADA symbol svg in preview mode --- .../dashboard-page/dashboard-widget-select.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-widget-select.component.html b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-widget-select.component.html index bfaaf700ce..216a7913a2 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-widget-select.component.html +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-widget-select.component.html @@ -51,7 +51,7 @@
- {{ item.title }} + {{ item.title }}
From 0947c1ba844ffbd178f21a011ac7d1799bf53272 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Wed, 27 Nov 2024 09:32:57 +0200 Subject: [PATCH 48/71] UI: Refactoring --- .../dashboard-page/dashboard-widget-select.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-widget-select.component.html b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-widget-select.component.html index 216a7913a2..f1bbca0292 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-widget-select.component.html +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-widget-select.component.html @@ -51,7 +51,7 @@
- {{ item.title }} + {{ item.title }}
From 01c45240f124bf4d93bae9054c0ade8c2f12cca7 Mon Sep 17 00:00:00 2001 From: thingsboard017 Date: Wed, 27 Nov 2024 11:02:28 +0200 Subject: [PATCH 49/71] Implemented timescale deprecated notification --- .../service/update/DeprecationService.java | 38 +++++++++++++++++++ .../src/main/resources/thingsboard.yml | 4 +- .../notification/DefaultNotifications.java | 10 +++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/update/DeprecationService.java b/application/src/main/java/org/thingsboard/server/service/update/DeprecationService.java index d4109ff420..539b5c58bb 100644 --- a/application/src/main/java/org/thingsboard/server/service/update/DeprecationService.java +++ b/application/src/main/java/org/thingsboard/server/service/update/DeprecationService.java @@ -26,6 +26,8 @@ import org.thingsboard.server.common.data.notification.targets.platform.SystemAd import org.thingsboard.server.dao.notification.DefaultNotifications; import org.thingsboard.server.queue.util.AfterStartUp; +import java.util.ArrayList; +import java.util.List; import java.util.Map; @Service @@ -38,9 +40,16 @@ public class DeprecationService { @Value("${queue.type}") private String queueType; + @Value("${database.ts.type}") + private String tsType; + + @Value("${database.ts_latest.type}") + private String tsLatestType; + @AfterStartUp(order = Integer.MAX_VALUE) public void checkDeprecation() { checkQueueTypeDeprecation(); + checkDatabaseTypeDeprecation(); } private void checkQueueTypeDeprecation() { @@ -63,4 +72,33 @@ public class DeprecationService { ))); } + private void checkDatabaseTypeDeprecation() { + String deprecatedDatabaseType = "timescale"; + var fieldsWithDeprecatedType = new ArrayList(); + + addFieldIfValueDeprecated(tsType, "ts", fieldsWithDeprecatedType, deprecatedDatabaseType); + addFieldIfValueDeprecated(tsLatestType, "ts_latest", fieldsWithDeprecatedType, deprecatedDatabaseType); + + if (fieldsWithDeprecatedType.isEmpty()) { + return; + } + + String deprecatedFieldNames = String.join(", ", fieldsWithDeprecatedType); + + log.warn("WARNING: Starting with ThingsBoard 4.0, {} will no longer be supported as a database type for telemetry storage. " + + "Please migrate to Cassandra or SQL-based storage solutions. This change will not impact the telemetry query.", + deprecatedDatabaseType); + + notificationCenter.sendGeneralWebNotification(TenantId.SYS_TENANT_ID, new SystemAdministratorsFilter(), + DefaultNotifications.databaseTypeDeprecation.toTemplate(), new GeneralNotificationInfo(Map.of( + "databaseType", deprecatedDatabaseType, + "deprecatedFields", deprecatedFieldNames + ))); + } + + private void addFieldIfValueDeprecated(String fieldValue, String fieldName, List fieldsWithDeprecatedType, String deprecatedDatabaseType) { + if (deprecatedDatabaseType.equals(fieldValue)) { + fieldsWithDeprecatedType.add(fieldName); + } + } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 2a121e2626..09d97b7e88 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -212,9 +212,9 @@ ui: database: ts_max_intervals: "${DATABASE_TS_MAX_INTERVALS:700}" # Max number of DB queries generated by a single API call to fetch telemetry records ts: - type: "${DATABASE_TS_TYPE:sql}" # cassandra, sql, or timescale (for hybrid mode, DATABASE_TS_TYPE value should be cassandra, or timescale) + type: "${DATABASE_TS_TYPE:timescale}" # cassandra, sql (timescale is deprecated and no longer supported in ThingsBoard 4.0) ts_latest: - type: "${DATABASE_TS_LATEST_TYPE:sql}" # cassandra, sql, or timescale (for hybrid mode, DATABASE_TS_TYPE value should be cassandra, or timescale) + type: "${DATABASE_TS_LATEST_TYPE:sql}" # cassandra, sql (timescale is deprecated and no longer supported in ThingsBoard 4.0) # Cassandra driver configuration parameters cassandra: diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java index 9b747766c9..6a904b5e64 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java @@ -381,6 +381,16 @@ public class DefaultNotifications { .icon("warning").color(RED_COLOR) .build(); + public static final DefaultNotification databaseTypeDeprecation = DefaultNotification.builder() + .name("Database type deprecation") + .type(NotificationType.GENERAL) + .subject("WARNING: Database type '${databaseType}' is deprecated") + .text("Starting with ThingsBoard 4.0, the database type '${databaseType}' (${deprecatedFields}) will no longer be supported. " + + "Please migrate to Cassandra or SQL-based storage solutions") + .icon("warning") + .color(RED_COLOR) + .build(); + private final NotificationTemplateService templateService; private final NotificationRuleService ruleService; From b07ccb374bf6020a55af50460a441e4df6ff4b7f Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 27 Nov 2024 09:56:20 +0200 Subject: [PATCH 50/71] UI: Updated dependencies to latest version --- ui-ngx/package.json | 83 +-- ....11.patch => @angular+build+18.2.12.patch} | 4 +- ...2.10.patch => @angular+core+18.2.13.patch} | 2 +- ...3.1.patch => jquery.terminal+2.44.1.patch} | 19 +- ui-ngx/yarn.lock | 698 +++++++++--------- 5 files changed, 406 insertions(+), 400 deletions(-) rename ui-ngx/patches/{@angular+build+18.2.11.patch => @angular+build+18.2.12.patch} (96%) rename ui-ngx/patches/{@angular+core+18.2.10.patch => @angular+core+18.2.13.patch} (97%) rename ui-ngx/patches/{jquery.terminal+2.43.1.patch => jquery.terminal+2.44.1.patch} (76%) diff --git a/ui-ngx/package.json b/ui-ngx/package.json index 0ac7b1e4e2..0eff3bbd8d 100644 --- a/ui-ngx/package.json +++ b/ui-ngx/package.json @@ -13,23 +13,22 @@ }, "private": true, "dependencies": { - "@angular/animations": "18.2.10", - "@angular/cdk": "18.2.11", - "@angular/common": "18.2.10", - "@angular/compiler": "18.2.10", - "@angular/core": "18.2.10", + "@angular/animations": "18.2.13", + "@angular/cdk": "18.2.14", + "@angular/common": "18.2.13", + "@angular/compiler": "18.2.13", + "@angular/core": "18.2.13", "@angular/flex-layout": "^15.0.0-beta.42", - "@angular/forms": "18.2.10", - "@angular/material": "18.2.11", - "@angular/platform-browser": "18.2.10", - "@angular/platform-browser-dynamic": "18.2.10", - "@angular/router": "18.2.10", + "@angular/forms": "18.2.13", + "@angular/material": "18.2.14", + "@angular/platform-browser": "18.2.13", + "@angular/platform-browser-dynamic": "18.2.13", + "@angular/router": "18.2.13", "@auth0/angular-jwt": "^5.2.0", "@emotion/react": "11.13.3", "@emotion/styled": "11.13.0", "@flowjs/flow.js": "^2.14.1", "@flowjs/ngx-flow": "18.0.1", - "@fortawesome/fontawesome-svg-core": "^6.6.0", "@geoman-io/leaflet-geoman-free": "2.17.0", "@iplab/ngx-color-picker": "^18.0.1", "@mat-datetimepicker/core": "~14.0.0", @@ -49,12 +48,12 @@ "@svgdotjs/svg.js": "^3.2.4", "@svgdotjs/svg.panzoom.js": "^2.1.2", "@tinymce/tinymce-angular": "^8.0.1", - "ace-builds": "1.36.2", + "ace-builds": "1.36.5", "ace-diff": "^3.0.3", "angular-gridster2": "~18.0.1", "angular2-hotkeys": "^16.0.1", "canvas-gauges": "^2.1.7", - "core-js": "^3.38.1", + "core-js": "^3.39.0", "dayjs": "1.11.13", "echarts": "https://github.com/thingsboard/echarts/archive/5.5.0-TB.tar.gz", "flot": "https://github.com/thingsboard/flot.git#0.9-work", @@ -62,7 +61,7 @@ "font-awesome": "^4.7.0", "html2canvas": "^1.4.1", "jquery": "^3.7.1", - "jquery.terminal": "^2.43.1", + "jquery.terminal": "^2.44.1", "js-beautify": "1.15.1", "json-schema-defaults": "^0.4.0", "jstree": "^3.3.17", @@ -73,7 +72,7 @@ "leaflet-providers": "2.0.0", "leaflet.gridlayer.googlemutant": "0.14.1", "leaflet.markercluster": "1.5.3", - "libphonenumber-js": "^1.11.10", + "libphonenumber-js": "^1.11.15", "marked": "~12.0.2", "moment": "^2.30.1", "moment-timezone": "^0.5.45", @@ -83,7 +82,7 @@ "ngx-flowchart": "https://github.com/thingsboard/ngx-flowchart.git#release/3.0.0", "ngx-hm-carousel": "^18.0.0", "ngx-markdown": "^18.1.0", - "ngx-sharebuttons": "^15.0.3", + "ngx-sharebuttons": "^15.0.6", "ngx-translate-messageformat-compiler": "^7.0.0", "objectpath": "^2.0.0", "prettier": "^2.8.3", @@ -112,60 +111,60 @@ }, "devDependencies": { "@angular-builders/custom-esbuild": "18.0.0", - "@angular-devkit/build-angular": "18.2.11", - "@angular-devkit/core": "18.2.11", - "@angular-devkit/schematics": "18.2.11", - "@angular-eslint/builder": "18.4.0", - "@angular-eslint/eslint-plugin": "18.4.0", - "@angular-eslint/eslint-plugin-template": "18.4.0", - "@angular-eslint/schematics": "18.4.0", - "@angular-eslint/template-parser": "18.4.0", - "@angular/build": "18.2.11", - "@angular/cli": "18.2.11", - "@angular/compiler-cli": "18.2.10", - "@angular/language-service": "18.2.10", + "@angular-devkit/build-angular": "18.2.12", + "@angular-devkit/core": "18.2.12", + "@angular-devkit/schematics": "18.2.12", + "@angular-eslint/builder": "18.4.2", + "@angular-eslint/eslint-plugin": "18.4.2", + "@angular-eslint/eslint-plugin-template": "18.4.2", + "@angular-eslint/schematics": "18.4.2", + "@angular-eslint/template-parser": "18.4.2", + "@angular/build": "18.2.12", + "@angular/cli": "18.2.12", + "@angular/compiler-cli": "18.2.13", + "@angular/language-service": "18.2.13", "@types/ace-diff": "^2.1.4", "@types/canvas-gauges": "^2.1.8", "@types/flot": "^0.0.36", "@types/flowjs": "^2.13.14", - "@types/jquery": "^3.5.31", + "@types/jquery": "^3.5.32", "@types/js-beautify": "^1.14.3", - "@types/leaflet": "1.9.12", + "@types/leaflet": "1.9.14", "@types/leaflet-polylinedecorator": "1.6.4", "@types/leaflet-providers": "1.2.4", "@types/leaflet.gridlayer.googlemutant": "0.4.9", - "@types/leaflet.markercluster": "1.5.4", - "@types/lodash": "^4.17.10", - "@types/node": "~18.15.11", + "@types/leaflet.markercluster": "1.5.5", + "@types/lodash": "^4.17.13", + "@types/node": "~20.17.8", "@types/raphael": "^2.3.9", "@types/react": "18.3.10", "@types/react-dom": "18.3.0", "@types/systemjs": "6.15.1", "@types/tinycolor2": "^1.4.6", "@types/tooltipster": "^0.0.35", - "@typescript-eslint/eslint-plugin": "^8.12.2", - "@typescript-eslint/parser": "^8.12.2", - "@typescript-eslint/types": "^8.12.2", - "@typescript-eslint/utils": "^8.12.2", + "@typescript-eslint/eslint-plugin": "^8.16.0", + "@typescript-eslint/parser": "^8.16.0", + "@typescript-eslint/types": "^8.16.0", + "@typescript-eslint/utils": "^8.16.0", "autoprefixer": "^10.4.20", "directory-tree": "^3.5.2", - "eslint": "~9.13.0", + "eslint": "~9.15.0", "eslint-plugin-import": "latest", - "eslint-plugin-jsdoc": "^50.4.3", + "eslint-plugin-jsdoc": "^50.6.0", "eslint-plugin-prefer-arrow": "latest", "eslint-plugin-tailwindcss": "^3.17.5", "ngrx-store-freeze": "^0.2.4", "patch-package": "^8.0.0", - "postcss": "^8.4.47", + "postcss": "^8.4.49", "postinstall-prepare": "^2.0.0", - "tailwindcss": "^3.4.14", + "tailwindcss": "^3.4.15", "ts-node": "^10.9.2", "typescript": "~5.5.4" }, "resolutions": { "@types/react": "18.3.10", "rc-virtual-list": "3.5.2", - "ace-builds": "1.36.2", + "ace-builds": "1.36.5", "tinymce": "6.8.5", "rollup": "4.22.4", "@babel/core": "7.25.2", diff --git a/ui-ngx/patches/@angular+build+18.2.11.patch b/ui-ngx/patches/@angular+build+18.2.12.patch similarity index 96% rename from ui-ngx/patches/@angular+build+18.2.11.patch rename to ui-ngx/patches/@angular+build+18.2.12.patch index 17b84e45c6..8861374691 100644 --- a/ui-ngx/patches/@angular+build+18.2.11.patch +++ b/ui-ngx/patches/@angular+build+18.2.12.patch @@ -12,7 +12,7 @@ index 625c621..4fc8bd8 100755 } async diagnoseFiles(modes = DiagnosticModes.All) { diff --git a/node_modules/@angular/build/src/tools/esbuild/angular/compiler-plugin.js b/node_modules/@angular/build/src/tools/esbuild/angular/compiler-plugin.js -index b1bb6ea..c76b4c6 100755 +index b03875c..8c48101 100755 --- a/node_modules/@angular/build/src/tools/esbuild/angular/compiler-plugin.js +++ b/node_modules/@angular/build/src/tools/esbuild/angular/compiler-plugin.js @@ -79,7 +79,7 @@ function createCompilerPlugin(pluginOptions, styleOptions) { @@ -24,7 +24,7 @@ index b1bb6ea..c76b4c6 100755 }, environment_options_1.maxWorkers, cacheStore?.createCache('jstransformer')); // Setup defines based on the values used by the Angular compiler-cli build.initialOptions.define ??= {}; -@@ -377,12 +377,14 @@ function createCompilerPlugin(pluginOptions, styleOptions) { +@@ -385,12 +385,14 @@ function createCompilerPlugin(pluginOptions, styleOptions) { async function hasSideEffects(path) { if (!pluginOptions.advancedOptimizations) { return undefined; diff --git a/ui-ngx/patches/@angular+core+18.2.10.patch b/ui-ngx/patches/@angular+core+18.2.13.patch similarity index 97% rename from ui-ngx/patches/@angular+core+18.2.10.patch rename to ui-ngx/patches/@angular+core+18.2.13.patch index 7779ea145b..c6d635e621 100644 --- a/ui-ngx/patches/@angular+core+18.2.10.patch +++ b/ui-ngx/patches/@angular+core+18.2.13.patch @@ -1,5 +1,5 @@ diff --git a/node_modules/@angular/core/fesm2022/core.mjs b/node_modules/@angular/core/fesm2022/core.mjs -index 6fe7ad6..291e4ed 100755 +index 7d9d345..b4d21c6 100755 --- a/node_modules/@angular/core/fesm2022/core.mjs +++ b/node_modules/@angular/core/fesm2022/core.mjs @@ -12859,13 +12859,13 @@ function findDirectiveDefMatches(tView, tNode) { diff --git a/ui-ngx/patches/jquery.terminal+2.43.1.patch b/ui-ngx/patches/jquery.terminal+2.44.1.patch similarity index 76% rename from ui-ngx/patches/jquery.terminal+2.43.1.patch rename to ui-ngx/patches/jquery.terminal+2.44.1.patch index f5c34ac212..640316d87c 100644 --- a/ui-ngx/patches/jquery.terminal+2.43.1.patch +++ b/ui-ngx/patches/jquery.terminal+2.44.1.patch @@ -1,14 +1,27 @@ +diff --git a/node_modules/jquery.terminal/css/jquery.terminal.css b/node_modules/jquery.terminal/css/jquery.terminal.css +index 996fe49..2e71336 100644 +--- a/node_modules/jquery.terminal/css/jquery.terminal.css ++++ b/node_modules/jquery.terminal/css/jquery.terminal.css +@@ -23,7 +23,7 @@ + } + .cmd .cmd-clipboard { + position: absolute !important; +- let: -16px; ++ letf: -16px; + left: calc(-16px / var(--pixel-density, 1)) !important; + top: 0 !important; + width: 16px; diff --git a/node_modules/jquery.terminal/css/jquery.terminal.min.css b/node_modules/jquery.terminal/css/jquery.terminal.min.css -index 830ba6e..978e57b 100644 +index c24c7ea..bf55845 100644 --- a/node_modules/jquery.terminal/css/jquery.terminal.min.css +++ b/node_modules/jquery.terminal/css/jquery.terminal.min.css @@ -13,7 +13,7 @@ * Released under the MIT license * - * Date: Sun, 08 Sep 2024 09:58:43 +0000 + * Date: Tue, 08 Oct 2024 20:29:54 +0000 - */.cmd .format,.cmd-prompt,.cmd-prompt div,.terminal .terminal-output .format{display:inline-block}.cmd,.terminal h1,.terminal h2,.terminal h3,.terminal h4,.terminal h5,.terminal h6,.terminal pre{margin:0}.cmd .cmd-clipboard{let:-16px;background:transparent!important;border:none!important;box-shadow:none!important;color:transparent!important;height:16px;height:calc(16px/var(--pixel-density, 1))!important;left:calc(-16px/var(--pixel-density, 1))!important;outline:none!important;overflow:hidden!important;padding:0!important;position:absolute!important;resize:none!important;text-indent:-9999em!important;top:0!important;top:calc(var(--cursor-line, 0)*var(--size, 1)*(14px/var(--pixel-density, 1)))!important;white-space:pre!important;width:16px;width:calc(16px/var(--pixel-density, 1))!important;z-index:1000!important}.visually-hidden{clip:rect(0 0 0 0);clip-path:inset(50%);height:1px;left:0;overflow:hidden;position:absolute;top:0;user-select:none;white-space:nowrap;width:1px}.cmd .cursor+span:empty,.cmd div.cmd-end-line span[data-text]:last-child,.cmd div.cmd-end-line span[data-text]:last-child span,.cmd span.cmd-end-line,.cmd span.cmd-end-line span,.cmd textarea{-webkit-touch-callout:none;-webkit-user-select:none;-khtml-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.terminal audio,.terminal canvas,.terminal img,.terminal object{cursor:default}.terminal{--base-background:var(--background,#000);--base-color:var(--color);contain:content;line-height:normal;position:relative}.terminal-scroller{scrollbar-gutter:stable;box-sizing:border-box;height:100%;overflow-x:hidden;overflow-y:auto;position:relative}terminal.terminal-temp{visibility:hidden}body.terminal{height:100%;min-height:100vh}html{height:100%}body.full-screen-terminal,body.terminal{height:100%;margin:0}body.full-screen-terminal .terminal{height:100%!important}body.full-screen-terminal{height:100%;height:var(--terminal-force-height,100dvh)}.terminal>div.terminal-fill{height:100%;min-height:100%}.terminal>.terminal-pixel{display:inline;height:1px;left:-2px;position:absolute;top:-2px;width:1px}.terminal>.terminal-font .terminal-resizer,.terminal>.terminal-resizer{border:none!important;bottom:0!important;height:100%!important;left:0!important;overflow:hidden!important;padding:0!important;pointer-events:none!important;position:absolute!important;right:0!important;top:0!important;width:100%!important;z-index:-1!important}.cmd{padding:0;position:relative;width:100%;z-index:300}.terminal .cmd,.terminal .cmd div{background:transparent!important}.terminal a[tabindex="1000"],.terminal a[tabindex="1000"]:active,.terminal a[tabindex="1000"]:focus{outline:none}.cmd.cmd.cmd .cmd-inverted,.cmd.cmd.cmd .inverted,.terminal .inverted,.terminal .terminal-inverted{background-color:#aaa!important;color:#000!important}.cmd a[href],.terminal .terminal-output>:not(.raw) a[href],.terminal.external a[href]{color:#37f;color:var(--link-color,#37f);cursor:pointer}.cmd a[href]:not(.terminal-inverted),.terminal .terminal-output>:not(.raw) a[href]:not(.terminal-inverted),.terminal.external a[href]:not(.terminal-inverted){--color:var(--link-color,#37f);text-shadow:0 0 calc(var(--glow)*5px) var(--color,#ccc)}.terminal .terminal-output>:not(.raw) a[href].terminal-inverted,.terminal.external a[href].terminal-inverted{background:var(--color,#ccc);text-shadow:0 0 calc(var(--glow)*5px) var(--background,#000)}.cmd a[href]:hover,.terminal .terminal-output>:not(.raw) a[href]:hover,.terminal.external a[href]:hover{background-color:#37f;background-color:var(--link-color,#37f)!important;color:#000;color:var(--base-background,#000)!important;text-decoration:none}.cmd a[href] span,.terminal .terminal-output>:not(.raw) a[href] span,.terminal.external a[href] span{--color:var(--link-color,#37f);color:#37f!important;color:var(--link-color,#37f)!important;text-decoration:underline}.cmd a[href]:hover span,.terminal .terminal-output>:not(.raw) a[href]:hover span,.terminal.external a[href]:hover span{background-color:#37f!important;background-color:var(--link-color,#37f)!important;color:#000!important;color:var(--background,#000)!important;text-decoration:none}.terminal a[href]{transition-duration:.2s;transition-property:color,background-color;transition-timing-function:ease-out}.cmd .cmd-cursor,.cmd .cmd-cursor-line img,.cmd .cmd-cursor-line>span{display:inline-block}.cmd .cmd-cursor .emoji,.cmd .cmd-cursor.cmd-blink .fa,.cmd .cmd-cursor.cmd-blink .far,.cmd .cmd-cursor.cmd-blink .fas,.cmd .cmd-cursor.cmd-blink a,.cmd .cmd-cursor.cmd-blink>span[data-text]:not(.emoji):not(.fa):not(.far):not(.fas) span{-webkit-animation:terminal-blink 1s linear infinite;-moz-animation:terminal-blink 1s linear infinite;-ms-animation:terminal-blink 1s linear infinite;animation:terminal-blink 1s linear infinite}.bar.cmd .cmd-inverted,.bar.terminal .inverted{box-shadow:-2px 0 0 -1px #aaa;box-shadow:-2px 0 0 -1px var(--original-color,#aaa)}.cmd .cmd-prompt,.terminal,.terminal .terminal-output>div>div{display:block;height:auto}.terminal .terminal-output>div:not(.raw) div{clear:both;white-space:nowrap}.cmd .cmd-prompt:empty,.cmd .cmd-prompt>a,.cmd .cmd-prompt>span{float:left}.cmd-prompt [data-text]>span,.cmd-wrapper>div{white-space:pre}.cmd [data-text] span,.terminal [data-text] span{display:inline-block}.terminal-ouput span[style*=width]{min-height:14px;min-height:calc(var(--size, 1)*(14px/var(--pixel-density, 1)))}.cmd div,.terminal .terminal-output>:not(.raw)>div,.terminal.external div{line-height:1em}.cmd .cmd-prompt span.fa:before,.cmd .cmd-prompt span.fab:before,.cmd .cmd-prompt span.fad:before,.cmd .cmd-prompt span.fal:before,.cmd .cmd-prompt span.far:before,.cmd .cmd-prompt span.fas{position:relative;top:2px}.cmd,.cmd span:not(.fas):not(.far):not(.fa),.terminal,.terminal-font-forcer,.terminal-output>:not(.raw),.terminal-output>:not(.raw) a,.terminal-output>:not(.raw) span:not(.fas):not(.far):not(.fa){font-family:monospace;font-family:var(--font,monospace)}.cmd,.terminal{font-size:12px}.cmd .cmd-line:before,.terminal-output>div:not(.raw) div:before{content:"\0200B";display:inline-block;width:0}.cmd .cmd-line{white-space:nowrap}.cmd span[data-text],.terminal span[data-text]{display:inline-block}.cmd span[data-text]:not(.cmd-inverted):not(.token):not(.emoji),.terminal,terminal-output>div:not(.raw) div>span:not(.token):not(.inverted):not(.terminal-inverted):not(.cmd-inverted):not(.terminal-error):not(.emoji){background-color:#000;color:#aaa}.cmd span[data-text] span,.terminal span[data-text] span{text-decoration:inherit}.terminal .ansi>div{line-height:13px!important;line-height:calc(var(--size, 1)*(13px/var(--pixel-density, 1)))!important}.cmd .cmd-prompt span,.cmd span.cmd-prompt{--background:transparent}.cmd .emoji,.terminal-output .emoji{background-repeat:no-repeat;background-size:contain;color:transparent;height:12px;height:calc(12px/var(--pixel-density, 1));position:relative}.cmd .fa span,.cmd .far span,.cmd .fas span,.terminal .terminal-output .fa span,.terminal .terminal-output .far span,.terminal .terminal-output .fas span{clip:rect(1px,1px,1px,1px);background:transparent!important;color:transparent!important;position:absolute}.cmd .emoji,.cmd .emoji span,.terminal-output .emoji,.terminal-output .emoji span{display:inline-block;width:2ch}.cmd,.terminal{box-sizing:border-box}.cmd .cmd-cursor span:not(.token):not(.inverted){background-color:inherit;color:inherit}.cmd .emoji.emoji.emoji.emoji,.cmd .emoji.emoji.emoji.emoji span{background-color:transparent;color:transparent}.cmd .cmd-cursor *{background-color:transparent}.cmd span[style*=width] span,.terminal span[style*=width] span{width:inherit}.cmd div{clear:both}.cmd .cmd-prompt+div{clear:right}terminal .terminal-output>div{margin-top:-1px}.terminal-output>div.raw>div *{word-wrap:break-word;overflow-wrap:break-word}.terminal .terminal-font{float:left;font-size:inherit;left:0;line-height:inherit;margin-bottom:1px;position:absolute;top:-100%}.cmd>span:not(.cmd-prompt){float:left}.cmd .cmd-prompt span.cmd-line{display:block;float:none}.terminal table{border-collapse:collapse}.terminal td{border:1px solid #aaa}.cmd span[data-text]:not(.emoji):not(.fa):not(.fas):not(.far) span{background-color:inherit;color:inherit}.cmd [role=presentation].cmd-cursor-line{cursor:text;position:relative;z-index:100}.cmd .cmd-prompt{position:relative;z-index:200}.cmd [role=presentation]:not(.cmd-cursor-line){overflow:hidden}.cmd{--original-color:var( + */.cmd .format,.cmd-prompt,.cmd-prompt div,.terminal .terminal-output .format{display:inline-block}.cmd,.terminal h1,.terminal h2,.terminal h3,.terminal h4,.terminal h5,.terminal h6,.terminal pre{margin:0}.cmd .cmd-clipboard{left:-16px;background:transparent!important;border:none!important;box-shadow:none!important;color:transparent!important;height:16px;height:calc(16px/var(--pixel-density, 1))!important;left:calc(-16px/var(--pixel-density, 1))!important;outline:none!important;overflow:hidden!important;padding:0!important;position:absolute!important;resize:none!important;text-indent:-9999em!important;top:0!important;top:calc(var(--cursor-line, 0)*var(--size, 1)*(14px/var(--pixel-density, 1)))!important;white-space:pre!important;width:16px;width:calc(16px/var(--pixel-density, 1))!important;z-index:1000!important}.visually-hidden{clip:rect(0 0 0 0);clip-path:inset(50%);height:1px;left:0;overflow:hidden;position:absolute;top:0;user-select:none;white-space:nowrap;width:1px}.cmd .cursor+span:empty,.cmd div.cmd-end-line span[data-text]:last-child,.cmd div.cmd-end-line span[data-text]:last-child span,.cmd span.cmd-end-line,.cmd span.cmd-end-line span,.cmd textarea{-webkit-touch-callout:none;-webkit-user-select:none;-khtml-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.terminal audio,.terminal canvas,.terminal img,.terminal object{cursor:default}.terminal{--base-background:var(--background,#000);--base-color:var(--color);contain:content;line-height:normal;position:relative}.terminal-scroller{scrollbar-gutter:stable;box-sizing:border-box;height:100%;overflow-x:hidden;overflow-y:auto;position:relative}terminal.terminal-temp{visibility:hidden}body.terminal{height:100%;min-height:100vh}html{height:100%}body.full-screen-terminal,body.terminal{height:100%;margin:0}body.full-screen-terminal .terminal{height:100%!important}body.full-screen-terminal{height:100%;height:var(--terminal-force-height,100dvh)}.terminal>div.terminal-fill{height:100%;min-height:100%}.terminal>.terminal-pixel{display:inline;height:1px;left:-2px;position:absolute;top:-2px;width:1px}.terminal>.terminal-font .terminal-resizer,.terminal>.terminal-resizer{border:none!important;bottom:0!important;height:100%!important;left:0!important;overflow:hidden!important;padding:0!important;pointer-events:none!important;position:absolute!important;right:0!important;top:0!important;width:100%!important;z-index:-1!important}.cmd{padding:0;position:relative;width:100%;z-index:300}.terminal .cmd,.terminal .cmd div{background:transparent!important}.terminal a[tabindex="1000"],.terminal a[tabindex="1000"]:active,.terminal a[tabindex="1000"]:focus{outline:none}.cmd.cmd.cmd .cmd-inverted,.cmd.cmd.cmd .inverted,.terminal .inverted,.terminal .terminal-inverted{background-color:#aaa!important;color:#000!important}.cmd a[href],.terminal .terminal-output>:not(.raw) a[href],.terminal.external a[href]{color:#37f;color:var(--link-color,#37f);cursor:pointer}.cmd a[href]:not(.terminal-inverted),.terminal .terminal-output>:not(.raw) a[href]:not(.terminal-inverted),.terminal.external a[href]:not(.terminal-inverted){--color:var(--link-color,#37f);text-shadow:0 0 calc(var(--glow)*5px) var(--color,#ccc)}.terminal .terminal-output>:not(.raw) a[href].terminal-inverted,.terminal.external a[href].terminal-inverted{background:var(--color,#ccc);text-shadow:0 0 calc(var(--glow)*5px) var(--background,#000)}.cmd a[href]:hover,.terminal .terminal-output>:not(.raw) a[href]:hover,.terminal.external a[href]:hover{background-color:#37f;background-color:var(--link-color,#37f)!important;color:#000;color:var(--base-background,#000)!important;text-decoration:none}.cmd a[href] span,.terminal .terminal-output>:not(.raw) a[href] span,.terminal.external a[href] span{--color:var(--link-color,#37f);color:#37f!important;color:var(--link-color,#37f)!important;text-decoration:underline}.cmd a[href]:hover span,.terminal .terminal-output>:not(.raw) a[href]:hover span,.terminal.external a[href]:hover span{background-color:#37f!important;background-color:var(--link-color,#37f)!important;color:#000!important;color:var(--background,#000)!important;text-decoration:none}.terminal a[href]{transition-duration:.2s;transition-property:color,background-color;transition-timing-function:ease-out}.cmd .cmd-cursor,.cmd .cmd-cursor-line img,.cmd .cmd-cursor-line>span{display:inline-block}.cmd .cmd-cursor .emoji,.cmd .cmd-cursor.cmd-blink .fa,.cmd .cmd-cursor.cmd-blink .far,.cmd .cmd-cursor.cmd-blink .fas,.cmd .cmd-cursor.cmd-blink a,.cmd .cmd-cursor.cmd-blink>span[data-text]:not(.emoji):not(.fa):not(.far):not(.fas) span{-webkit-animation:terminal-blink 1s linear infinite;-moz-animation:terminal-blink 1s linear infinite;-ms-animation:terminal-blink 1s linear infinite;animation:terminal-blink 1s linear infinite}.bar.cmd .cmd-inverted,.bar.terminal .inverted{box-shadow:-2px 0 0 -1px #aaa;box-shadow:-2px 0 0 -1px var(--original-color,#aaa)}.cmd .cmd-prompt,.terminal,.terminal .terminal-output>div>div{display:block;height:auto}.terminal .terminal-output>div:not(.raw) div{clear:both;white-space:nowrap}.cmd .cmd-prompt:empty,.cmd .cmd-prompt>a,.cmd .cmd-prompt>span{float:left}.cmd-prompt [data-text]>span,.cmd-wrapper>div{white-space:pre}.cmd [data-text] span,.terminal [data-text] span{display:inline-block}.terminal-ouput span[style*=width]{min-height:14px;min-height:calc(var(--size, 1)*(14px/var(--pixel-density, 1)))}.cmd div,.terminal .terminal-output>:not(.raw)>div,.terminal.external div{line-height:1em}.cmd .cmd-prompt span.fa:before,.cmd .cmd-prompt span.fab:before,.cmd .cmd-prompt span.fad:before,.cmd .cmd-prompt span.fal:before,.cmd .cmd-prompt span.far:before,.cmd .cmd-prompt span.fas{position:relative;top:2px}.cmd,.cmd span:not(.fas):not(.far):not(.fa),.terminal,.terminal-font-forcer,.terminal-output>:not(.raw),.terminal-output>:not(.raw) a,.terminal-output>:not(.raw) span:not(.fas):not(.far):not(.fa){font-family:monospace;font-family:var(--font,monospace)}.cmd,.terminal{font-size:12px}.cmd .cmd-line:before,.terminal-output>div:not(.raw) div:before{content:"\0200B";display:inline-block;width:0}.cmd .cmd-line{white-space:nowrap}.cmd span[data-text],.terminal span[data-text]{display:inline-block}.cmd span[data-text]:not(.cmd-inverted):not(.token):not(.emoji),.terminal,terminal-output>div:not(.raw) div>span:not(.token):not(.inverted):not(.terminal-inverted):not(.cmd-inverted):not(.terminal-error):not(.emoji){background-color:#000;color:#aaa}.cmd span[data-text] span,.terminal span[data-text] span{text-decoration:inherit}.terminal .ansi>div{line-height:13px!important;line-height:calc(var(--size, 1)*(13px/var(--pixel-density, 1)))!important}.cmd .cmd-prompt span,.cmd span.cmd-prompt{--background:transparent}.cmd .emoji,.terminal-output .emoji{background-repeat:no-repeat;background-size:contain;color:transparent;height:12px;height:calc(12px/var(--pixel-density, 1));position:relative}.cmd .fa span,.cmd .far span,.cmd .fas span,.terminal .terminal-output .fa span,.terminal .terminal-output .far span,.terminal .terminal-output .fas span{clip:rect(1px,1px,1px,1px);background:transparent!important;color:transparent!important;position:absolute}.cmd .emoji,.cmd .emoji span,.terminal-output .emoji,.terminal-output .emoji span{display:inline-block;width:2ch}.cmd,.terminal{box-sizing:border-box}.cmd .cmd-cursor span:not(.token):not(.inverted){background-color:inherit;color:inherit}.cmd .emoji.emoji.emoji.emoji,.cmd .emoji.emoji.emoji.emoji span{background-color:transparent;color:transparent}.cmd .cmd-cursor *{background-color:transparent}.cmd span[style*=width] span,.terminal span[style*=width] span{width:inherit}.cmd div{clear:both}.cmd .cmd-prompt+div{clear:right}terminal .terminal-output>div{margin-top:-1px}.terminal-output>div.raw>div *{word-wrap:break-word;overflow-wrap:break-word}.terminal .terminal-font{float:left;font-size:inherit;left:0;line-height:inherit;margin-bottom:1px;position:absolute;top:-100%}.cmd>span:not(.cmd-prompt){float:left}.cmd .cmd-prompt span.cmd-line{display:block;float:none}.terminal table{border-collapse:collapse}.terminal td{border:1px solid #aaa}.cmd span[data-text]:not(.emoji):not(.fa):not(.fas):not(.far) span{background-color:inherit;color:inherit}.cmd [role=presentation].cmd-cursor-line{cursor:text;position:relative;z-index:100}.cmd .cmd-prompt{position:relative;z-index:200}.cmd [role=presentation]:not(.cmd-cursor-line){overflow:hidden}.cmd{--original-color:var( --color,#aaa);--original-background:var( - --background)}.cmd a[href]{--original-color:var(--link-color,#37f)}@-webkit-keyframes terminal-blink{0%,50%{background-color:#aaa;background-color:var(--original-color,#aaa);-webkit-box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);color:#000;color:var(--original-background,#000)}50.1%,to{background-color:inherit;color:inherit;color:var(--original-color,#aaa)}}@-moz-keyframes terminal-blink{0%,50%{background-color:#aaa;background-color:var(--original-color,#aaa);-webkit-box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);color:#000;color:var(--original-background,#000)}50.1%,to{background-color:inherit;color:inherit;color:var(--original-color,#aaa)}}@keyframes terminal-blink{0%,50%{background-color:#aaa;background-color:var(--original-color,#aaa);-webkit-box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);color:#000;color:var(--original-background,#000)}50.1%,to{background-color:inherit;-webkit-box-shadow:none;box-shadow:none;color:inherit;color:var(--original-color,#aaa)}}@-webkit-keyframes terminal-glow{0%,50%{background-color:#aaa;background-color:var(--original-color,#aaa);box-shadow:0 0 3px #aaa;-webkit-box-shadow:0 0 3px var(--color,#aaa);box-shadow:0 0 3px var(--color,#aaa);color:#000;color:var(--original-background,#000)}50.1%,to{background-color:inherit;-webkit-box-shadow:none;box-shadow:none;color:inherit}}@-moz-keyframes terminal-glow{0%,50%{background-color:#aaa;background-color:var(--original-color,#aaa);box-shadow:0 0 3px #aaa;-moz-box-shadow:0 0 3px var(--color,#aaa);box-shadow:0 0 3px var(--color,#aaa);color:#000;color:var(--original-background,#000)}50.1%,to{background-color:inherit;-moz-box-shadow:none;box-shadow:none;color:inherit}}@keyframes terminal-glow{0%,50%{background-color:#aaa;background-color:var(--original-color,#aaa);box-shadow:0 0 3px #aaa;box-shadow:0 0 3px var(--color,#aaa);color:#000;color:var(--original-background,#000)}50.1%,to{background-color:inherit;box-shadow:none;color:inherit}}@-webkit-keyframes terminal-bar{0%,50%{box-shadow:-2px 0 0 -1px #aaa;box-shadow:calc(var(--line-thickness, 2)*-1px) 0 0 -1px var(--original-color,#aaa)}50.1%,to{box-shadow:none}}@-moz-keyframes terminal-bar{0%,50%{box-shadow:-2px 0 0 -1px #aaa;box-shadow:calc(var(--line-thickness, 2)*-1px) 0 0 -1px var(--original-color,#aaa)}50.1%,to{box-shadow:none}}@keyframes terminal-bar{0%,50%{box-shadow:-2px 0 0 -1px #aaa;box-shadow:calc(var(--line-thickness, 2)*-1px) 0 0 -1px var(--original-color,#aaa)}50.1%,to{box-shadow:none}}@-webkit-keyframes terminal-underline{0%,50%{box-shadow:0 2px 0 #aaa;box-shadow:0 calc(var(--line-thickness, 2)*1px) 0 var(--original-color,#aaa)}50.1%,to{box-shadow:none}}@-moz-keyframes terminal-underline{0%,50%{box-shadow:0 2px 0 #aaa;box-shadow:0 calc(var(--line-thickness, 2)*1px) 0 var(--original-color,#aaa)}50.1%,to{box-shadow:none}}@keyframes terminal-underline{0%,50%{box-shadow:0 2px 0 #aaa;box-shadow:0 calc(var(--line-thickness, 2)*1px) 0 var(--original-color,#aaa)}50.1%,to{box-shadow:none}}@-webkit-keyframes terminal-none{0%,to{background-color:#aaa;background-color:var(--original-color,#aaa);-webkit-box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);color:#000;color:var(--original-background,#000)}}@-moz-keyframes terminal-none{0%,to{background-color:#aaa;background-color:var(--original-color,#aaa);-webkit-box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);color:#000;color:var(--background,#000)}}@keyframes terminal-none{0%,to{background-color:#aaa;background-color:var(--original-color,#aaa);-webkit-box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);color:#000;color:var(--original-background,#000)}}.underline-animation .cmd .cmd-cursor .emoji,.underline-animation .cmd .cmd-cursor.cmd-blink .fa,.underline-animation .cmd .cmd-cursor.cmd-blink .far,.underline-animation .cmd .cmd-cursor.cmd-blink .fas,.underline-animation .cmd .cmd-cursor.cmd-blink a,.underline-animation .cmd .cmd-cursor.cmd-blink>span[data-text]:not(.emoji):not(.fa):not(.far):not(.fas) span{-webkit-animation-name:terminal-underline;-moz-animation-name:terminal-underline;-ms-animation-name:terminal-underline;animation-name:terminal-underline}.glow-animation .cmd .cmd-cursor .emoji,.glow-animation .cmd .cmd-cursor.cmd-blink .fa,.glow-animation .cmd .cmd-cursor.cmd-blink .far,.glow-animation .cmd .cmd-cursor.cmd-blink .fas,.glow-animation .cmd .cmd-cursor.cmd-blink a,.glow-animation .cmd .cmd-cursor.cmd-blink>span[data-text]:not(.emoji):not(.fa):not(.far):not(.fas) span{-webkit-animation-name:terminal-glow;-moz-animation-name:terminal-glow;-ms-animation-name:terminal-glow;animation-name:terminal-glow}.bar-animation .cmd .cmd-cursor .emoji,.bar-animation .cmd .cmd-cursor.cmd-blink .fa,.bar-animation .cmd .cmd-cursor.cmd-blink .far,.bar-animation .cmd .cmd-cursor.cmd-blink .fas,.bar-animation .cmd .cmd-cursor.cmd-blink a,.bar-animation .cmd .cmd-cursor.cmd-blink>span[data-text]:not(.emoji):not(.fa):not(.far):not(.fas) span{-webkit-animation-name:terminal-bar;-moz-animation-name:terminal-bar;-ms-animation-name:terminal-bar;animation-name:terminal-bar}@supports (-ms-ime-align:auto){.cmd .cmd-clipboard{margin-left:-9999px}@keyframes terminal-blink{0%,50%{background-color:var(--original-color,#aaa);color:var(--background,#000)}50.1%,to{background-color:var(--background,#000);color:var(--original-color,#aaa)}}@keyframes terminal-bar{0%,50%{border-left-color:var(--color,#aaa)}50.1%,to{border-left-color:var(--background,#000)}}@keyframes terminal-underline{0%,50%{border-bottom-color:var(--color,#aaa);line-height:12px;line-height:calc(var(--size, 1)*(12px/var(--pixel-density, 1)))}50.1%,to{border-bottom-color:var(--background,#000);line-height:12px;line-height:calc(var(--size, 1)*(12px/var(--pixel-density, 1)))}}}@media (-ms-high-contrast:active),(-ms-high-contrast:none){.cmd .cmd-clipboard{margin-left:-9999px}.underline-animation .cursor.blink span span{margin-top:1px}@-ms-keyframes terminal-blink{0%,50%{background-color:#aaa;color:#000}50.1%,to{background-color:#000;color:#aaa}}}.cmd span[data-text]::-moz-selection,.cmd span[data-text]:not(.far):not(.fa):not(.fas) span::-moz-selection,.terminal .terminal-output .raw ::-moz-selection,.terminal .terminal-output .raw div::-moz-selection,.terminal .terminal-output div div a::-moz-selection,.terminal .terminal-output div div::-moz-selection,.terminal .terminal-output span[data-text]::-moz-selection,.terminal .terminal-output span[data-text]:not(.far):not(.fa):not(.fas) span::-moz-selection{background-color:#aaa;color:#000}.terminal .cmd{pointer-events:none}.terminal .cmd-prompt *{pointer-events:visible}.cmd-prompt>span:first-child{margin-left:var(--prompt-offset)}.terminal .cmd textarea,.terminal .cmd-cursor-line>span,.terminal .cmd-wrapper>div:not(.cmd-cursor-line){pointer-events:visible}.cmd span[data-text]:not(.far):not(.fa):not(.fas) span::selection,.terminal .terminal-output .raw ::selection,.terminal .terminal-output .raw div::selection,.terminal .terminal-output div div a::selection,.terminal .terminal-output div div::selection,.terminal .terminal-output span[data-text]::selection,.terminal .terminal-output span[data-text]:not(.far):not(.fa):not(.fas) span::selection,.terminal-revese{background-color:hsla(0,0%,67%,.99);color:#000}.cmd .emoji span::-moz-selection,.cmd .emoji::-moz-selection,.cmd textarea::-moz-selection,.terminal-output .emoji span::-moz-selection,.terminal-output .emoji::-moz-selection{background-color:transparent!important;color:transparent!important}.cmd .emoji span::selection,.cmd .emoji::selection,.cmd textarea::selection,.terminal-output .emoji span::selection,.terminal-output .emoji::selection{background-color:transparent!important;color:transparent!important}.terminal .terminal-output>:not(.raw) .terminal-error,.terminal .terminal-output>:not(.raw) .terminal-error *{color:red;color:var(--error-color,red)}.tilda{left:0;position:fixed;top:0;width:100%;z-index:1100}.ui-dialog-content .terminal{box-sizing:border-box;height:100%;width:100%}.ui-dialog .ui-dialog-content.dterm{padding:0}.clear{clear:both}.terminal .terminal-fill{border:none;box-sizing:border-box;height:100%;left:0;margin:1px 0 0;opacity:.01;pointer-events:none;position:absolute;top:-100%;width:100%}.cmd-editable,.terminal .terminal-fill,.terminal-scroller{padding:calc(1px*var(--padding, 10)/var(--pixel-density, 1))}.cmd-editable{padding-top:0}.terminal-scroller{padding-bottom:0}.terminal .terminal-output>:not(.raw) .terminal-error,.terminal .terminal-output>:not(.raw) .terminal-error *{--color:var(--error-color,red);text-shadow:0 0 calc(var(--glow)*5px) var(--color)}.terminal .terminal-output>:not(.raw) a[href]{text-shadow:0 0 calc(var(--glow)*5px) var(--color)}.terminal .cmd{margin-bottom:calc(1px*var(--padding, 10)/var(--pixel-density, 1));position:relative}.terminal .partial,.terminal .partial>div{display:inline-block}@supports (--css:variables){.cmd,.cmd div,.cmd span[data-text]:not(.cmd-inverted):not(.token):not(.emoji),.terminal,.terminal-output>:not(.raw) a,.terminal-output>:not(.raw) div,.terminal-output>:not(.raw) span[data-text]:not(.token):not(.inverted):not(.terminal-inverted):not(.cmd-inverted):not(.terminal-error):not(.emoji),.terminal.external [data-text]{background-color:var(--background,#000);color:var(--color,#aaa)}.terminal{--terminal-line:calc(var(--size, 1)*(16px/var(--pixel-density, 1)) + 1px/var(--pixel-density, 1));height:auto;height:calc(var(--terminal-line)*var(--rows) + var(--padding, 10)*2px)}.cmd div,.terminal .terminal-output>:not(.raw)>div,.terminal.external div{line-height:var(--terminal-line,1em)}.cmd [data-text],.terminal [data-text]{text-shadow:var(--text-shadow,0 0 calc(var(--glow)*5px) var(--color,#ccc))}.terminal span[style*="--length"]{display:inline-block;width:calc(var(--length, 1)*var(--char-width, 7.23438)*1px)}.cmd,.cmd div,.cmd span,.terminal,.terminal-output>:not(.raw) a,.terminal-output>:not(.raw) div,.terminal-output>:not(.raw) span{font-size:calc(var(--size, 1)*(12px/var(--pixel-density, 1)))}.cmd .emoji,.terminal-output .emoji{height:calc(var(--size, 1)*(12px/var(--pixel-density, 1)))}.cmd .clipboard{top:calc(var(--size, 1)*(14/var(--pixel-density, 1))*var(--cursor-line, 0)*1px)}.cmd span[style*="--background"],.terminal span[style*="--background"]{--base-background:var(--background)}.cmd.cmd.cmd .cmd-inverted,.cmd.cmd.cmd .inverted,.terminal .inverted>span,.terminal .terminal-inverted>span{background-color:var(--color,#aaa)!important;color:var(--base-background,#000)!important}.cmd .cmd-cursor.cmd-blink{background-color:var(--background,#000);color:var(--color,#aaa)}.cmd .cmd-cursor.cmd-blink .emoji,.cmd .cmd-cursor.cmd-blink .fa,.cmd .cmd-cursor.cmd-blink .far,.cmd .cmd-cursor.cmd-blink .fas,.cmd .cmd-cursor.cmd-blink a,.cmd .cmd-cursor.cmd-blink>span[data-text]:not(.emoji):not(.fa):not(.far):not(.fas) span{--original-background:inherit;-webkit-animation:var(--animation,terminal-blink) 1s infinite linear;-moz-animation:var(--animation,terminal-blink) 1s infinite linear;-ms-animation:var(--animation,terminal-blink) 1s infinite linear;animation:var(--animation,terminal-blink) 1s infinite linear}.cmd .cmd-cursor.cmd-blink .emoji span{background:transparent;color:transparent}.cmd span[data-text]:not(.far):not(.fa):not(.fas):not(.emoji) span::-moz-selection,.terminal .terminal-output .raw ::-moz-selection,.terminal .terminal-output .raw div::-moz-selection,.terminal .terminal-output div div a::-moz-selection,.terminal .terminal-output div div::-moz-selection,.terminal .terminal-output span[data-text]::-moz-selection,.terminal .terminal-output span[data-text]:not(.far):not(.fa):not(.fas):not(.emoji) span::-moz-selection{background-color:var(--color,#aaa);color:var(--background,#000)}.terminal .terminal-output div div a::-moz-selection{background-color:var(--link-color,rgba(15,96,255,.99))!important;color:var(--background,#000)!important}.terminal .terminal-output div div a:hover::-moz-selection{background-color:var(--link-color,rgba(2,50,144,.99))!important}.cmd span[data-text]:not(.far):not(.fa):not(.fas):not(.emoji) span::selection,.terminal .terminal-output .raw ::selection,.terminal .terminal-output .raw div::selection,.terminal .terminal-output div div a::selection,.terminal .terminal-output div div::selection,.terminal .terminal-output span[data-text]:not(.emoji)::selection,.terminal .terminal-output span[data-text]:not(.far):not(.fa):not(.fas):not(.emoji) span::selection{background-color:var(--color,hsla(0,0%,67%,.99))!important;color:var(--background,#000)!important}.terminal .terminal-output div div a::selection{background-color:var(--link-color,rgba(15,96,255,.99))!important;color:var(--background,#000)!important}.terminal .terminal-output div div a:hover::selection{background-color:var(--link-color,rgba(2,50,144,.99))!important}}@property --color{syntax:"";inherits:true;initial-value:hsla(0,0%,67%,.99)}@property --background{syntax:"";inherits:true;initial-value:#000}@property --link-color{syntax:"";inherits:true;initial-value:#37f}@property --size{syntax:"";inherits:true;initial-value:1}@property --padding{syntax:"";inherits:true;initial-value:10}@property --rows{syntax:"";inherits:true}@supports (-ms-ime-align:auto){.cmd span[data-text]::selection,.terminal .terminal-output div div a::selection,.terminal .terminal-output div div::selection,.terminal h1::selection,.terminal h2::selection,.terminal h3::selection,.terminal h4::selection,.terminal h5::selection,.terminal h6::selection,.terminal pre::selection,.terminal td::selection{background-color:hsla(0,0%,67%,.99);color:#000}}.cmd .style .token.string,.cmd .token.entity,.cmd .token.operator,.cmd .token.string,.cmd .token.token,.cmd .token.url,.cmd .token.variable,.terminal .style .token.string,.terminal .token.entity,.terminal .token.operator,.terminal .token.string,.terminal .token.token,.terminal .token.url,.terminal .token.variable{background-color:inherit}.cmd .cursor-wrapper ul{float:left;left:0;list-style:none;margin:0;padding:0;position:absolute;top:14px}.cmd .cursor-wrapper li{cursor:pointer;white-space:nowrap}.cmd .cursor-wrapper li:hover{background:#aaa;color:#000}.cursor-wrapper{position:relative}.terminal-output img{vertical-align:top}.cmd img{border:1px solid transparent;height:14px;height:calc(var(--size, 1)*(14px/var(--pixel-density, 1)))}.cmd-cursor img{border-color:#ccc;border-color:var(--color,#ccc)}.terminal-output svg.terminal-broken-image{height:calc(var(--size, 1)*(14px/var(--pixel-density, 1)))}.terminal-output svg.terminal-broken-image use{fill:var(--color,#ccc)}.terminal-error{--color:var(--error-color)}.terminal-glow{--animation:terminal-glow}.terminal-glow .cmd-prompt>span,.terminal-glow .terminal-output>div a[href],.terminal-glow .terminal-output>div span,.terminal-glow [data-text] span,.terminal-glow a[data-text],.terminal-glow span[data-text]{text-shadow:1px 1px 5px #ccc;text-shadow:1px 1px 5px var(--color,#ccc)}.terminal-scroll-marker{height:1px;height:calc(1px/var(--pixel-density, 1));margin-top:-1px;margin-top:calc(-1px/var(--pixel-density, 1));position:relative;z-index:100}.terminal-scroll-marker div{bottom:0;left:0;position:absolute;right:0;z-index:200}.terminal-less{overscroll-behavior-y:contain;touch-action:none}.terminal-mobile.terminal-less .terminal-wrapper{pointer-events:none}.cmd-editable,.terminal-mobile.terminal-less .terminal-output a{pointer-events:visible}.cmd-editable:before{content:attr(data-cmd-prompt);display:inline-block}.cmd-editable{background:transparent;bottom:0;color:transparent;left:0;opacity:.01;outline:0 solid transparent;position:absolute;right:0;top:calc(var(--cmd-top)*1px);z-index:500}.terminal-scroller::-webkit-scrollbar{background:var(--background,#000);height:calc(6px/var(--pixel-density, 1));width:calc(6px/var(--pixel-density, 1))}.terminal-scroller::-webkit-scrollbar-thumb,.terminal-scroller::-webkit-scrollbar-thumb:hover{background:var(--color,#aaa)}.terminal-scroller{scrollbar-color:#aaa #000;scrollbar-color:var(--color,#aaa) var(--background,#000);scrollbar-width:thin}.terminal-hidden{clip:rect(0,0,0,0)!important;border:0!important;height:1px!important;margin:-1px!important;overflow:hidden!important;padding:0!important;position:absolute!important;white-space:nowrap!important;width:1px!important}.terminal-wrapper{--color:var(--base-color);--background:var(--base-background)}.cmd .token{--original-color:var(--color)}.cmd .terminal-blink,.terminal .terminal-blink{animation:terminal-ansi-blink 1s steps(2,start) infinite;-webkit-animation:terminal-ansi-blink 1s steps(2,start) infinite}.terminal .cmd *,.terminal .terminal-output,.terminal .terminal-output *{--background:transparent}.terminal .cmd ::selection,.terminal .cmd::selection,.terminal .terminal-output ::selection,.terminal .terminal-output::selection{--background:var(--base-background)}@keyframes terminal-ansi-blink{to{color:var(--background)}}@-webkit-keyframes terminal-ansi-blink{to{color:var(--background)}}@media (prefers-reduced-motion){:root{--animation:terminal-none}} + --background)}.cmd a[href]{--original-color:var(--link-color,#37f)}@-webkit-keyframes terminal-blink{0%,50%{background-color:#aaa;background-color:var(--original-color,#aaa);-webkit-box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);color:#000;color:var(--original-background,#000)}50.1%,to{background-color:inherit;color:inherit;color:var(--original-color,#aaa)}}@-moz-keyframes terminal-blink{0%,50%{background-color:#aaa;background-color:var(--original-color,#aaa);-webkit-box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);color:#000;color:var(--original-background,#000)}50.1%,to{background-color:inherit;color:inherit;color:var(--original-color,#aaa)}}@keyframes terminal-blink{0%,50%{background-color:#aaa;background-color:var(--original-color,#aaa);-webkit-box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);color:#000;color:var(--original-background,#000)}50.1%,to{background-color:inherit;-webkit-box-shadow:none;box-shadow:none;color:inherit;color:var(--original-color,#aaa)}}@-webkit-keyframes terminal-glow{0%,50%{background-color:#aaa;background-color:var(--original-color,#aaa);box-shadow:0 0 3px #aaa;-webkit-box-shadow:0 0 3px var(--color,#aaa);box-shadow:0 0 3px var(--color,#aaa);color:#000;color:var(--original-background,#000)}50.1%,to{background-color:inherit;-webkit-box-shadow:none;box-shadow:none;color:inherit}}@-moz-keyframes terminal-glow{0%,50%{background-color:#aaa;background-color:var(--original-color,#aaa);box-shadow:0 0 3px #aaa;-moz-box-shadow:0 0 3px var(--color,#aaa);box-shadow:0 0 3px var(--color,#aaa);color:#000;color:var(--original-background,#000)}50.1%,to{background-color:inherit;-moz-box-shadow:none;box-shadow:none;color:inherit}}@keyframes terminal-glow{0%,50%{background-color:#aaa;background-color:var(--original-color,#aaa);box-shadow:0 0 3px #aaa;box-shadow:0 0 3px var(--color,#aaa);color:#000;color:var(--original-background,#000)}50.1%,to{background-color:inherit;box-shadow:none;color:inherit}}@-webkit-keyframes terminal-bar{0%,50%{box-shadow:-2px 0 0 -1px #aaa;box-shadow:calc(var(--line-thickness, 2)*-1px) 0 0 -1px var(--original-color,#aaa)}50.1%,to{box-shadow:none}}@-moz-keyframes terminal-bar{0%,50%{box-shadow:-2px 0 0 -1px #aaa;box-shadow:calc(var(--line-thickness, 2)*-1px) 0 0 -1px var(--original-color,#aaa)}50.1%,to{box-shadow:none}}@keyframes terminal-bar{0%,50%{box-shadow:-2px 0 0 -1px #aaa;box-shadow:calc(var(--line-thickness, 2)*-1px) 0 0 -1px var(--original-color,#aaa)}50.1%,to{box-shadow:none}}@-webkit-keyframes terminal-underline{0%,50%{box-shadow:0 2px 0 #aaa;box-shadow:0 calc(var(--line-thickness, 2)*1px) 0 var(--original-color,#aaa)}50.1%,to{box-shadow:none}}@-moz-keyframes terminal-underline{0%,50%{box-shadow:0 2px 0 #aaa;box-shadow:0 calc(var(--line-thickness, 2)*1px) 0 var(--original-color,#aaa)}50.1%,to{box-shadow:none}}@keyframes terminal-underline{0%,50%{box-shadow:0 2px 0 #aaa;box-shadow:0 calc(var(--line-thickness, 2)*1px) 0 var(--original-color,#aaa)}50.1%,to{box-shadow:none}}@-webkit-keyframes terminal-none{0%,to{background-color:#aaa;background-color:var(--original-color,#aaa);-webkit-box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);color:#000;color:var(--original-background,#000)}}@-moz-keyframes terminal-none{0%,to{background-color:#aaa;background-color:var(--original-color,#aaa);-webkit-box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);color:#000;color:var(--background,#000)}}@keyframes terminal-none{0%,to{background-color:#aaa;background-color:var(--original-color,#aaa);-webkit-box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);box-shadow:0 0 calc(var(--glow)*3px) var(--color,#aaa);color:#000;color:var(--original-background,#000)}}.underline-animation .cmd .cmd-cursor .emoji,.underline-animation .cmd .cmd-cursor.cmd-blink .fa,.underline-animation .cmd .cmd-cursor.cmd-blink .far,.underline-animation .cmd .cmd-cursor.cmd-blink .fas,.underline-animation .cmd .cmd-cursor.cmd-blink a,.underline-animation .cmd .cmd-cursor.cmd-blink>span[data-text]:not(.emoji):not(.fa):not(.far):not(.fas) span{-webkit-animation-name:terminal-underline;-moz-animation-name:terminal-underline;-ms-animation-name:terminal-underline;animation-name:terminal-underline}.glow-animation .cmd .cmd-cursor .emoji,.glow-animation .cmd .cmd-cursor.cmd-blink .fa,.glow-animation .cmd .cmd-cursor.cmd-blink .far,.glow-animation .cmd .cmd-cursor.cmd-blink .fas,.glow-animation .cmd .cmd-cursor.cmd-blink a,.glow-animation .cmd .cmd-cursor.cmd-blink>span[data-text]:not(.emoji):not(.fa):not(.far):not(.fas) span{-webkit-animation-name:terminal-glow;-moz-animation-name:terminal-glow;-ms-animation-name:terminal-glow;animation-name:terminal-glow}.bar-animation .cmd .cmd-cursor .emoji,.bar-animation .cmd .cmd-cursor.cmd-blink .fa,.bar-animation .cmd .cmd-cursor.cmd-blink .far,.bar-animation .cmd .cmd-cursor.cmd-blink .fas,.bar-animation .cmd .cmd-cursor.cmd-blink a,.bar-animation .cmd .cmd-cursor.cmd-blink>span[data-text]:not(.emoji):not(.fa):not(.far):not(.fas) span{-webkit-animation-name:terminal-bar;-moz-animation-name:terminal-bar;-ms-animation-name:terminal-bar;animation-name:terminal-bar}@supports (-ms-ime-align:auto){.cmd .cmd-clipboard{margin-left:-9999px}@keyframes terminal-blink{0%,50%{background-color:var(--original-color,#aaa);color:var(--background,#000)}50.1%,to{background-color:var(--background,#000);color:var(--original-color,#aaa)}}@keyframes terminal-bar{0%,50%{border-left-color:var(--color,#aaa)}50.1%,to{border-left-color:var(--background,#000)}}@keyframes terminal-underline{0%,50%{border-bottom-color:var(--color,#aaa);line-height:12px;line-height:calc(var(--size, 1)*(12px/var(--pixel-density, 1)))}50.1%,to{border-bottom-color:var(--background,#000);line-height:12px;line-height:calc(var(--size, 1)*(12px/var(--pixel-density, 1)))}}}@media (-ms-high-contrast:active),(-ms-high-contrast:none){.cmd .cmd-clipboard{margin-left:-9999px}.underline-animation .cursor.blink span span{margin-top:1px}@-ms-keyframes terminal-blink{0%,50%{background-color:#aaa;color:#000}50.1%,to{background-color:#000;color:#aaa}}}.cmd span[data-text]::-moz-selection,.cmd span[data-text]:not(.far):not(.fa):not(.fas) span::-moz-selection,.terminal .terminal-output .raw ::-moz-selection,.terminal .terminal-output .raw div::-moz-selection,.terminal .terminal-output div div a::-moz-selection,.terminal .terminal-output div div::-moz-selection,.terminal .terminal-output span[data-text]::-moz-selection,.terminal .terminal-output span[data-text]:not(.far):not(.fa):not(.fas) span::-moz-selection{background-color:#aaa;color:#000}.terminal .cmd{pointer-events:none}.terminal .cmd-prompt *{pointer-events:visible}.cmd-prompt>span:first-child{margin-left:var(--prompt-offset)}.terminal .cmd textarea,.terminal .cmd-cursor-line>span,.terminal .cmd-wrapper>div:not(.cmd-cursor-line){pointer-events:visible}.cmd span[data-text]:not(.far):not(.fa):not(.fas) span::selection,.terminal .terminal-output .raw ::selection,.terminal .terminal-output .raw div::selection,.terminal .terminal-output div div a::selection,.terminal .terminal-output div div::selection,.terminal .terminal-output span[data-text]::selection,.terminal .terminal-output span[data-text]:not(.far):not(.fa):not(.fas) span::selection,.terminal-revese{background-color:hsla(0,0%,67%,.99);color:#000}.cmd .emoji span::-moz-selection,.cmd .emoji::-moz-selection,.cmd textarea::-moz-selection,.terminal-output .emoji span::-moz-selection,.terminal-output .emoji::-moz-selection{background-color:transparent!important;color:transparent!important}.cmd .emoji span::selection,.cmd .emoji::selection,.cmd textarea::selection,.terminal-output .emoji span::selection,.terminal-output .emoji::selection{background-color:transparent!important;color:transparent!important}.terminal .terminal-output>:not(.raw) .terminal-error,.terminal .terminal-output>:not(.raw) .terminal-error *{color:red;color:var(--error-color,red)}.tilda{left:0;position:fixed;top:0;width:100%;z-index:1100}.ui-dialog-content .terminal{box-sizing:border-box;height:100%;width:100%}.ui-dialog .ui-dialog-content.dterm{padding:0}.clear{clear:both}.terminal .terminal-fill{border:none;box-sizing:border-box;height:100%;left:0;margin:1px 0 0;opacity:.01;pointer-events:none;position:absolute;top:-100%;width:100%}.cmd-editable,.terminal .terminal-fill,.terminal-scroller{padding:calc(1px*var(--padding, 10)/var(--pixel-density, 1))}.cmd-editable{padding-top:0}.terminal-scroller{padding-bottom:0}.terminal .terminal-output>:not(.raw) .terminal-error,.terminal .terminal-output>:not(.raw) .terminal-error *{--color:var(--error-color,red);text-shadow:0 0 calc(var(--glow)*5px) var(--color)}.terminal .terminal-output>:not(.raw) a[href]{text-shadow:0 0 calc(var(--glow)*5px) var(--color)}.terminal .cmd{margin-bottom:calc(1px*var(--padding, 10)/var(--pixel-density, 1));position:relative}.terminal .partial,.terminal .partial>div{display:inline-block}@supports (--css:variables){.cmd,.cmd div,.cmd span[data-text]:not(.cmd-inverted):not(.token):not(.emoji),.terminal,.terminal-output>:not(.raw) a,.terminal-output>:not(.raw) div,.terminal-output>:not(.raw) span[data-text]:not(.token):not(.inverted):not(.terminal-inverted):not(.cmd-inverted):not(.terminal-error):not(.emoji),.terminal.external [data-text]{background-color:var(--background,#000);color:var(--color,#aaa)}.terminal{--terminal-line:calc(var(--size, 1)*(16px/var(--pixel-density, 1)) + 1px/var(--pixel-density, 1));height:auto;height:calc(var(--terminal-line)*var(--rows) + var(--padding, 10)*2px)}.cmd div,.terminal,.terminal .terminal-output>:not(.raw)>div,.terminal.external div{line-height:round(var(--terminal-line,1em))}.cmd [data-text],.terminal [data-text]{text-shadow:var(--text-shadow,0 0 calc(var(--glow)*5px) var(--color,#ccc))}.terminal span[style*="--length"]{display:inline-block;width:calc(var(--length, 1)*var(--char-width, 7.23438)*1px)}.cmd,.cmd div,.cmd span,.terminal,.terminal-output>:not(.raw) a,.terminal-output>:not(.raw) div,.terminal-output>:not(.raw) span{font-size:calc(var(--size, 1)*(12px/var(--pixel-density, 1)))}.cmd .emoji,.terminal-output .emoji{height:calc(var(--size, 1)*(12px/var(--pixel-density, 1)))}.cmd .clipboard{top:calc(var(--size, 1)*(14/var(--pixel-density, 1))*var(--cursor-line, 0)*1px)}.cmd span[style*="--background"],.terminal span[style*="--background"]{--base-background:var(--background)}.cmd.cmd.cmd .cmd-inverted,.cmd.cmd.cmd .inverted,.terminal .inverted>span,.terminal .terminal-inverted>span{background-color:var(--color,#aaa)!important;color:var(--base-background,#000)!important}.cmd .cmd-cursor.cmd-blink{background-color:var(--background,#000);color:var(--color,#aaa)}.cmd .cmd-cursor.cmd-blink .emoji,.cmd .cmd-cursor.cmd-blink .fa,.cmd .cmd-cursor.cmd-blink .far,.cmd .cmd-cursor.cmd-blink .fas,.cmd .cmd-cursor.cmd-blink a,.cmd .cmd-cursor.cmd-blink>span[data-text]:not(.emoji):not(.fa):not(.far):not(.fas) span{--original-background:inherit;-webkit-animation:var(--animation,terminal-blink) 1s infinite linear;-moz-animation:var(--animation,terminal-blink) 1s infinite linear;-ms-animation:var(--animation,terminal-blink) 1s infinite linear;animation:var(--animation,terminal-blink) 1s infinite linear}.cmd .cmd-cursor.cmd-blink .emoji span{background:transparent;color:transparent}.cmd span[data-text]:not(.far):not(.fa):not(.fas):not(.emoji) span::-moz-selection,.terminal .terminal-output .raw ::-moz-selection,.terminal .terminal-output .raw div::-moz-selection,.terminal .terminal-output div div a::-moz-selection,.terminal .terminal-output div div::-moz-selection,.terminal .terminal-output span[data-text]::-moz-selection,.terminal .terminal-output span[data-text]:not(.far):not(.fa):not(.fas):not(.emoji) span::-moz-selection{background-color:var(--color,#aaa);color:var(--background,#000)}.terminal .terminal-output div div a::-moz-selection{background-color:var(--link-color,rgba(15,96,255,.99))!important;color:var(--background,#000)!important}.terminal .terminal-output div div a:hover::-moz-selection{background-color:var(--link-color,rgba(2,50,144,.99))!important}.cmd span[data-text]:not(.far):not(.fa):not(.fas):not(.emoji) span::selection,.terminal .terminal-output .raw ::selection,.terminal .terminal-output .raw div::selection,.terminal .terminal-output div div a::selection,.terminal .terminal-output div div::selection,.terminal .terminal-output span[data-text]:not(.emoji)::selection,.terminal .terminal-output span[data-text]:not(.far):not(.fa):not(.fas):not(.emoji) span::selection{background-color:var(--color,hsla(0,0%,67%,.99))!important;color:var(--background,#000)!important}.terminal .terminal-output div div a::selection{background-color:var(--link-color,rgba(15,96,255,.99))!important;color:var(--background,#000)!important}.terminal .terminal-output div div a:hover::selection{background-color:var(--link-color,rgba(2,50,144,.99))!important}}@property --color{syntax:"";inherits:true;initial-value:hsla(0,0%,67%,.99)}@property --background{syntax:"";inherits:true;initial-value:#000}@property --link-color{syntax:"";inherits:true;initial-value:#37f}@property --size{syntax:"";inherits:true;initial-value:1}@property --padding{syntax:"";inherits:true;initial-value:10}@property --rows{syntax:" | auto";inherits:true;initial-value:auto}@supports (-ms-ime-align:auto){.cmd span[data-text]::selection,.terminal .terminal-output div div a::selection,.terminal .terminal-output div div::selection,.terminal h1::selection,.terminal h2::selection,.terminal h3::selection,.terminal h4::selection,.terminal h5::selection,.terminal h6::selection,.terminal pre::selection,.terminal td::selection{background-color:hsla(0,0%,67%,.99);color:#000}}.cmd .style .token.string,.cmd .token.entity,.cmd .token.operator,.cmd .token.string,.cmd .token.token,.cmd .token.url,.cmd .token.variable,.terminal .style .token.string,.terminal .token.entity,.terminal .token.operator,.terminal .token.string,.terminal .token.token,.terminal .token.url,.terminal .token.variable{background-color:inherit}.cmd .cursor-wrapper ul{float:left;left:0;list-style:none;margin:0;padding:0;position:absolute;top:14px}.cmd .cursor-wrapper li{cursor:pointer;white-space:nowrap}.cmd .cursor-wrapper li:hover{background:#aaa;color:#000}.cursor-wrapper{position:relative}.terminal-output img{vertical-align:top}.cmd img{border:1px solid transparent;height:14px;height:calc(var(--size, 1)*(14px/var(--pixel-density, 1)))}.cmd-cursor img{border-color:#ccc;border-color:var(--color,#ccc)}.terminal-output svg.terminal-broken-image{height:calc(var(--size, 1)*(14px/var(--pixel-density, 1)))}.terminal-output svg.terminal-broken-image use{fill:var(--color,#ccc)}.terminal-error{--color:var(--error-color)}.terminal-glow{--animation:terminal-glow}.terminal-glow .cmd-prompt>span,.terminal-glow .terminal-output>div a[href],.terminal-glow .terminal-output>div span,.terminal-glow [data-text] span,.terminal-glow a[data-text],.terminal-glow span[data-text]{text-shadow:1px 1px 5px #ccc;text-shadow:1px 1px 5px var(--color,#ccc)}.terminal-scroll-marker{height:1px;height:calc(1px/var(--pixel-density, 1));margin-top:-1px;margin-top:calc(-1px/var(--pixel-density, 1));position:relative;z-index:100}.terminal-scroll-marker div{bottom:0;left:0;position:absolute;right:0;z-index:200}.terminal-less{overscroll-behavior-y:contain;touch-action:none}.terminal-mobile.terminal-less .terminal-wrapper{pointer-events:none}.cmd-editable,.terminal-mobile.terminal-less .terminal-output a{pointer-events:visible}.cmd-editable:before{content:attr(data-cmd-prompt);display:inline-block}.cmd-editable{background:transparent;bottom:0;color:transparent;left:0;opacity:.01;outline:0 solid transparent;position:absolute;right:0;top:calc(var(--cmd-top)*1px);z-index:500}.terminal-scroller::-webkit-scrollbar{background:var(--background,#000);height:calc(6px/var(--pixel-density, 1));width:calc(6px/var(--pixel-density, 1))}.terminal-scroller::-webkit-scrollbar-thumb,.terminal-scroller::-webkit-scrollbar-thumb:hover{background:var(--color,#aaa)}.terminal-scroller{scrollbar-color:#aaa #000;scrollbar-color:var(--color,#aaa) var(--background,#000);scrollbar-width:thin}.terminal-hidden{clip:rect(0,0,0,0)!important;border:0!important;height:1px!important;margin:-1px!important;overflow:hidden!important;padding:0!important;position:absolute!important;white-space:nowrap!important;width:1px!important}.terminal-wrapper{--color:var(--base-color);--background:var(--base-background)}.cmd .token{--original-color:var(--color)}.cmd .terminal-blink,.terminal .terminal-blink{animation:terminal-ansi-blink 1s steps(2,start) infinite;-webkit-animation:terminal-ansi-blink 1s steps(2,start) infinite}.terminal .cmd *,.terminal .terminal-output,.terminal .terminal-output *{--background:transparent}.terminal .cmd ::selection,.terminal .cmd::selection,.terminal .terminal-output ::selection,.terminal .terminal-output::selection{--background:var(--base-background)}@keyframes terminal-ansi-blink{to{color:var(--background)}}@-webkit-keyframes terminal-ansi-blink{to{color:var(--background)}}@media (prefers-reduced-motion){:root{--animation:terminal-none}} /*# sourceMappingURL=jquery.terminal.min.css.map */ \ No newline at end of file diff --git a/ui-ngx/yarn.lock b/ui-ngx/yarn.lock index c51b279dce..b7e5e21e55 100644 --- a/ui-ngx/yarn.lock +++ b/ui-ngx/yarn.lock @@ -34,24 +34,24 @@ "@angular-devkit/build-angular" "^18.0.0" "@angular-devkit/core" "^18.0.0" -"@angular-devkit/architect@0.1802.11", "@angular-devkit/architect@>=0.1800.0 < 0.1900.0": - version "0.1802.11" - resolved "https://registry.yarnpkg.com/@angular-devkit/architect/-/architect-0.1802.11.tgz#c1c53fa40aa26ce9fa5a0cc66bb401ac5a9306f0" - integrity sha512-p+XIc/j51aI83ExNdeZwvkm1F4wkuKMGUUoj0MVUUi5E6NoiMlXYm6uU8+HbRvPBzGy5+3KOiGp3Fks0UmDSAA== +"@angular-devkit/architect@0.1802.12", "@angular-devkit/architect@>=0.1800.0 < 0.1900.0": + version "0.1802.12" + resolved "https://registry.yarnpkg.com/@angular-devkit/architect/-/architect-0.1802.12.tgz#096f8e9cf71f8848c6f0172c03f3f1135509e133" + integrity sha512-bepVb2/GtJppYKaeW8yTGE6egmoWZ7zagFDsmBdbF+BYp+HmeoPsclARcdryBPVq68zedyTRdvhWSUTbw1AYuw== dependencies: - "@angular-devkit/core" "18.2.11" + "@angular-devkit/core" "18.2.12" rxjs "7.8.1" -"@angular-devkit/build-angular@18.2.11", "@angular-devkit/build-angular@^18.0.0": - version "18.2.11" - resolved "https://registry.yarnpkg.com/@angular-devkit/build-angular/-/build-angular-18.2.11.tgz#96766d409559074c9abf8c2691b7e189911896e6" - integrity sha512-09Ln3NAdlMw/wMLgnwYU5VgWV5TPBEHolZUIvE9D8b6SFWBCowk3B3RWeAMgg7Peuf9SKwqQHBz2b1C7RTP/8g== +"@angular-devkit/build-angular@18.2.12", "@angular-devkit/build-angular@^18.0.0": + version "18.2.12" + resolved "https://registry.yarnpkg.com/@angular-devkit/build-angular/-/build-angular-18.2.12.tgz#1ab09bc5571a1fc8c6234ef346f0629761c27bdd" + integrity sha512-quVUi7eqTq9OHumQFNl9Y8t2opm8miu4rlYnuF6rbujmmBDvdUvR6trFChueRczl2p5HWqTOr6NPoDGQm8AyNw== dependencies: "@ampproject/remapping" "2.3.0" - "@angular-devkit/architect" "0.1802.11" - "@angular-devkit/build-webpack" "0.1802.11" - "@angular-devkit/core" "18.2.11" - "@angular/build" "18.2.11" + "@angular-devkit/architect" "0.1802.12" + "@angular-devkit/build-webpack" "0.1802.12" + "@angular-devkit/core" "18.2.12" + "@angular/build" "18.2.12" "@babel/core" "7.25.2" "@babel/generator" "7.25.0" "@babel/helper-annotate-as-pure" "7.24.7" @@ -62,7 +62,7 @@ "@babel/preset-env" "7.25.3" "@babel/runtime" "7.25.0" "@discoveryjs/json-ext" "0.6.1" - "@ngtools/webpack" "18.2.11" + "@ngtools/webpack" "18.2.12" "@vitejs/plugin-basic-ssl" "1.1.0" ansi-colors "4.1.3" autoprefixer "10.4.20" @@ -112,18 +112,18 @@ optionalDependencies: esbuild "0.23.0" -"@angular-devkit/build-webpack@0.1802.11": - version "0.1802.11" - resolved "https://registry.yarnpkg.com/@angular-devkit/build-webpack/-/build-webpack-0.1802.11.tgz#15399456b0a95454564953baab41f5c7c0924bce" - integrity sha512-G76rNsyn1iQk7qjyr+K4rnDzfalmEswmwXQorypSDGaHYzIDY1SZXMoP4225WMq5fJNBOJrk82FA0PSfnPE+zQ== +"@angular-devkit/build-webpack@0.1802.12": + version "0.1802.12" + resolved "https://registry.yarnpkg.com/@angular-devkit/build-webpack/-/build-webpack-0.1802.12.tgz#b46afdf8dccef681156118d545e37ac57e0da759" + integrity sha512-0Z3fdbZVRnjYWE2/VYyfy+uieY+6YZyEp4ylzklVkc+fmLNsnz4Zw6cK1LzzcBqAwKIyh1IdW20Cg7o8b0sONA== dependencies: - "@angular-devkit/architect" "0.1802.11" + "@angular-devkit/architect" "0.1802.12" rxjs "7.8.1" -"@angular-devkit/core@18.2.11", "@angular-devkit/core@^18.0.0": - version "18.2.11" - resolved "https://registry.yarnpkg.com/@angular-devkit/core/-/core-18.2.11.tgz#5f0ed261db0b658dbd502e024a4e2abee47fe272" - integrity sha512-H9P1shRGigORWJHUY2BRa2YurT+DVminrhuaYHsbhXBRsPmgB2Dx/30YLTnC1s5XmR9QIRUCsg/d3kyT1wd5Zg== +"@angular-devkit/core@18.2.12", "@angular-devkit/core@^18.0.0": + version "18.2.12" + resolved "https://registry.yarnpkg.com/@angular-devkit/core/-/core-18.2.12.tgz#fb514e9b3c9ea87ddaa1582d3947f1b094c9b387" + integrity sha512-NtB6ypsaDyPE6/fqWOdfTmACs+yK5RqfH5tStEzWFeeDsIEDYKsJ06ypuRep7qTjYus5Rmttk0Ds+cFgz8JdUQ== dependencies: ajv "8.17.1" ajv-formats "3.0.1" @@ -132,85 +132,85 @@ rxjs "7.8.1" source-map "0.7.4" -"@angular-devkit/schematics@18.2.11": - version "18.2.11" - resolved "https://registry.yarnpkg.com/@angular-devkit/schematics/-/schematics-18.2.11.tgz#5e3a9303d27286d0dd0e642d7ccaba5b7e3899a0" - integrity sha512-efRK3FotTFp4KD5u42jWfXpHUALXB9kJNsWiB4wEImKFH6CN+vjBspJQuLqk2oeBFh/7D2qRMc5P+2tZHM5hdw== +"@angular-devkit/schematics@18.2.12": + version "18.2.12" + resolved "https://registry.yarnpkg.com/@angular-devkit/schematics/-/schematics-18.2.12.tgz#15d1a8611bf9f18215435604672411b1929bf4d1" + integrity sha512-mMea9txHbnCX5lXLHlo0RAgfhFHDio45/jMsREM2PA8UtVf2S8ltXz7ZwUrUyMQRv8vaSfn4ijDstF4hDMnRgQ== dependencies: - "@angular-devkit/core" "18.2.11" + "@angular-devkit/core" "18.2.12" jsonc-parser "3.3.1" magic-string "0.30.11" ora "5.4.1" rxjs "7.8.1" -"@angular-eslint/builder@18.4.0": - version "18.4.0" - resolved "https://registry.yarnpkg.com/@angular-eslint/builder/-/builder-18.4.0.tgz#e4ce3a68bef10885dcc76898f09510cfddb9ce4b" - integrity sha512-FOzGHX/nHSV1wSduSsabsx3aqC1nfde0opEpEDSOJhxExDxKCwoS1XPy1aERGyKip4ZVA6phC3dLtoBH3QMkVQ== +"@angular-eslint/builder@18.4.2": + version "18.4.2" + resolved "https://registry.yarnpkg.com/@angular-eslint/builder/-/builder-18.4.2.tgz#28a4833919ede3db0a1d905fd903485588282115" + integrity sha512-eyI9sreaM9ukA24PCJoSqsjCYOiBf3TZ/Q1WY8PG0SwQWc03qJNqPl5K+/Ptmsc1RtoDCLCU6uaOBFPhb9lDxw== -"@angular-eslint/bundled-angular-compiler@18.4.0": - version "18.4.0" - resolved "https://registry.yarnpkg.com/@angular-eslint/bundled-angular-compiler/-/bundled-angular-compiler-18.4.0.tgz#3503d8b796a3d9d6c997b342a3ae5ddf264601a4" - integrity sha512-HlFHt2qgdd+jqyVIkCXmrjHauXo/XY3Rp0UNabk83ejGi/raM/6lEFI7iFWzHxLyiAKk4OgGI5W26giSQw991A== +"@angular-eslint/bundled-angular-compiler@18.4.2": + version "18.4.2" + resolved "https://registry.yarnpkg.com/@angular-eslint/bundled-angular-compiler/-/bundled-angular-compiler-18.4.2.tgz#3d9827d6aea627e77a93b223af1dc56a3ea5e258" + integrity sha512-K7pqmZI3Dl75zlLexyaM7bw4xdgk/3bhP1B6uqDKML9+vIIvccCR2bGvqFurqeFbJlMykzb3H4jytT+HpqV4tg== -"@angular-eslint/eslint-plugin-template@18.4.0": - version "18.4.0" - resolved "https://registry.yarnpkg.com/@angular-eslint/eslint-plugin-template/-/eslint-plugin-template-18.4.0.tgz#cc952f0643025bac23667a69c82887d44c6ab789" - integrity sha512-n3uZFCy76DnggPqjSVFV3gYD1ik7jCG28o2/HO4kobcMNKnwW8XAlFUagQ4TipNQh7fQiAefsEqvv2quMsYDVw== +"@angular-eslint/eslint-plugin-template@18.4.2": + version "18.4.2" + resolved "https://registry.yarnpkg.com/@angular-eslint/eslint-plugin-template/-/eslint-plugin-template-18.4.2.tgz#5cc70866b7d15fcc85f11cf497283969d3a98c9b" + integrity sha512-v9msmIdZK6lOEC4ScDeYKFLpszpJ5Ei+8ifkT7fXXKmPaWtPJtMbW+VGOUNm5Ezi+xByAGCn1qU+OF2aJ/4CLw== dependencies: - "@angular-eslint/bundled-angular-compiler" "18.4.0" - "@angular-eslint/utils" "18.4.0" + "@angular-eslint/bundled-angular-compiler" "18.4.2" + "@angular-eslint/utils" "18.4.2" aria-query "5.3.2" axobject-query "4.1.0" -"@angular-eslint/eslint-plugin@18.4.0": - version "18.4.0" - resolved "https://registry.yarnpkg.com/@angular-eslint/eslint-plugin/-/eslint-plugin-18.4.0.tgz#6b38e89e323f347a796c1918fdbb19a07d380318" - integrity sha512-Saz9lkWPN3da7ZKW17UsOSN7DeY+TPh+wz/6GCNZCh67Uw2wvMC9agb+4hgpZNXYCP5+u7erqzxQmBoWnS/A+A== +"@angular-eslint/eslint-plugin@18.4.2": + version "18.4.2" + resolved "https://registry.yarnpkg.com/@angular-eslint/eslint-plugin/-/eslint-plugin-18.4.2.tgz#c64b016f404521175c9484145f814787c2a31c00" + integrity sha512-Oem4W2P54cPADN9rJenLj90rqDPUQWx5kZiz84FCnsSn5DBdsI5LGQoogNT9y3Jx/9VL/VGIMMA5B6qG+0hVlg== dependencies: - "@angular-eslint/bundled-angular-compiler" "18.4.0" - "@angular-eslint/utils" "18.4.0" + "@angular-eslint/bundled-angular-compiler" "18.4.2" + "@angular-eslint/utils" "18.4.2" -"@angular-eslint/schematics@18.4.0": - version "18.4.0" - resolved "https://registry.yarnpkg.com/@angular-eslint/schematics/-/schematics-18.4.0.tgz#4da9110b88d26b21f7e6edb1d5f6cb310cb7eec6" - integrity sha512-ssqe+0YCfekbWIXNdCrHfoPK/bPZAWybs0Bn/b99dfd8h8uyXkERo9AzIOx4Uyj/08SkP9aPL/0uOOEHDsRGwQ== +"@angular-eslint/schematics@18.4.2": + version "18.4.2" + resolved "https://registry.yarnpkg.com/@angular-eslint/schematics/-/schematics-18.4.2.tgz#2dceb682958faceaf7bdf48cf5484d1546544d56" + integrity sha512-pZCc3NhfwRT5S0DGXTzKbl3dD4I8K4LRYot+Aq4rzY5LtiGHDSi4PKu2M0OBSRrQFQXq7/2gDXGO0AvH6LX97w== dependencies: - "@angular-eslint/eslint-plugin" "18.4.0" - "@angular-eslint/eslint-plugin-template" "18.4.0" - ignore "5.3.2" + "@angular-eslint/eslint-plugin" "18.4.2" + "@angular-eslint/eslint-plugin-template" "18.4.2" + ignore "6.0.2" semver "7.6.3" strip-json-comments "3.1.1" -"@angular-eslint/template-parser@18.4.0": - version "18.4.0" - resolved "https://registry.yarnpkg.com/@angular-eslint/template-parser/-/template-parser-18.4.0.tgz#92d0cc51d709b81a0589cb2c4f248c797ccec529" - integrity sha512-VTep3Xd3IOaRIPL+JN/TV4/2DqUPbjtF3TNY15diD/llnrEhqFnmsvMihexbQyTqzOG+zU554oK44YfvAtHOrw== +"@angular-eslint/template-parser@18.4.2": + version "18.4.2" + resolved "https://registry.yarnpkg.com/@angular-eslint/template-parser/-/template-parser-18.4.2.tgz#9e7d75b53ac8c50bfffeb01dadb297846ebbbed8" + integrity sha512-KGjDLUxMsdjaxC+8VTxCG07Q6qshOTWMYTvp2LZ4QBySDQnQuFwsIJIJfU8jJwzJCkPKfVpnyuHggAn7fdYnxA== dependencies: - "@angular-eslint/bundled-angular-compiler" "18.4.0" + "@angular-eslint/bundled-angular-compiler" "18.4.2" eslint-scope "^8.0.2" -"@angular-eslint/utils@18.4.0": - version "18.4.0" - resolved "https://registry.yarnpkg.com/@angular-eslint/utils/-/utils-18.4.0.tgz#f11d66bd6fe6b65509348117f6556fc30a516b67" - integrity sha512-At1yS8GRviGBoaupiQwEOL4/IcZJCE/+2vpXdItMWPGB1HWetxlKAUZTMmIBX/r5Z7CoXxl+LbqpGhrhyzIQAg== +"@angular-eslint/utils@18.4.2": + version "18.4.2" + resolved "https://registry.yarnpkg.com/@angular-eslint/utils/-/utils-18.4.2.tgz#65486d64ba0a6f67fa6c9658812492ed74a1a2e2" + integrity sha512-+c0r33QSkAnGmu/DYAPfzJJk5QDX4TP2d6EFtsenrufqRkZqrOcK4Q5t61J92Ukkr03XoqTzTDSBjlwAfM56Rw== dependencies: - "@angular-eslint/bundled-angular-compiler" "18.4.0" + "@angular-eslint/bundled-angular-compiler" "18.4.2" -"@angular/animations@18.2.10": - version "18.2.10" - resolved "https://registry.yarnpkg.com/@angular/animations/-/animations-18.2.10.tgz#3e14f5762c8b3735d7a8ad333cea47e0ac10da56" - integrity sha512-LT5+CocFZJ4t5jXsXLx5w/sBuWSxOEjmNTYga13usRcLOblrAB902pjUdFCHEZyrCUgm4MH8vov9fMS+Ks2GCw== +"@angular/animations@18.2.13": + version "18.2.13" + resolved "https://registry.yarnpkg.com/@angular/animations/-/animations-18.2.13.tgz#cbfec085fe68179ef2e559ea43a9f7efa95979fd" + integrity sha512-rG5J5Ek5Hg+Tz2NjkNOaG6PupiNK/lPfophXpsR1t/nWujqnMWX2krahD/i6kgD+jNWNKCJCYSOVvCx/BHOtKA== dependencies: tslib "^2.3.0" -"@angular/build@18.2.11": - version "18.2.11" - resolved "https://registry.yarnpkg.com/@angular/build/-/build-18.2.11.tgz#ff2bb4f01e98757922a63bb97d718dcbf767f13f" - integrity sha512-AgirvSCmqUKiDE3C0rl3JA68OkOqQWDKUvjqRHXCkhxldLVOVoeIl87+jBYK/v9gcmk+K+ju+5wbGEfu1FjhiQ== +"@angular/build@18.2.12": + version "18.2.12" + resolved "https://registry.yarnpkg.com/@angular/build/-/build-18.2.12.tgz#d6be89fb572675cfa170f664c0b0ed49c317f33c" + integrity sha512-4Ohz+OSILoL+cCAQ4UTiCT5v6pctu3fXNoNpTEUK46OmxELk9jDITO5rNyNS7TxBn9wY69kjX5VcDf7MenquFQ== dependencies: "@ampproject/remapping" "2.3.0" - "@angular-devkit/architect" "0.1802.11" + "@angular-devkit/architect" "0.1802.12" "@babel/core" "7.25.2" "@babel/helper-annotate-as-pure" "7.24.7" "@babel/helper-split-export-declaration" "7.24.7" @@ -235,26 +235,26 @@ vite "5.4.6" watchpack "2.4.1" -"@angular/cdk@18.2.11": - version "18.2.11" - resolved "https://registry.yarnpkg.com/@angular/cdk/-/cdk-18.2.11.tgz#aeaa9dbf7da51a607d88dd8288c17221db12d4ab" - integrity sha512-FuvfhrSz2ch0gyOVHrkWq2C/I2PnOzKYSXlG/VEG+ize/WNrvlYy//5WVrTh/hv+HD9sdoWPr9ULXsfFfgbo7w== +"@angular/cdk@18.2.14": + version "18.2.14" + resolved "https://registry.yarnpkg.com/@angular/cdk/-/cdk-18.2.14.tgz#6114f37a77f2f182de482da2273c70f74a6cc52d" + integrity sha512-vDyOh1lwjfVk9OqoroZAP8pf3xxKUvyl+TVR8nJxL4c5fOfUFkD7l94HaanqKSRwJcI2xiztuu92IVoHn8T33Q== dependencies: tslib "^2.3.0" optionalDependencies: parse5 "^7.1.2" -"@angular/cli@18.2.11": - version "18.2.11" - resolved "https://registry.yarnpkg.com/@angular/cli/-/cli-18.2.11.tgz#07bc4f797b68946ed9900f8be1ce5a76d0f19b67" - integrity sha512-0JI1xjOLRemBPjdT/yVlabxc3Zkjqa/lhvVxxVC1XhKoW7yGxIGwNrQ4pka4CcQtCuktO6KPMmTGIu8YgC3cpw== +"@angular/cli@18.2.12": + version "18.2.12" + resolved "https://registry.yarnpkg.com/@angular/cli/-/cli-18.2.12.tgz#ea1af54d1fd172bf8bd3c7ed977a422c1c1a94d9" + integrity sha512-xhuZ/b7IhqNw1MgXf+arWf4x+GfUSt/IwbdWU4+CO8A7h0Y46zQywouP/KUK3cMQZfVdHdciTBvlpF3vFacA6Q== dependencies: - "@angular-devkit/architect" "0.1802.11" - "@angular-devkit/core" "18.2.11" - "@angular-devkit/schematics" "18.2.11" + "@angular-devkit/architect" "0.1802.12" + "@angular-devkit/core" "18.2.12" + "@angular-devkit/schematics" "18.2.12" "@inquirer/prompts" "5.3.8" "@listr2/prompt-adapter-inquirer" "2.0.15" - "@schematics/angular" "18.2.11" + "@schematics/angular" "18.2.12" "@yarnpkg/lockfile" "1.1.0" ini "4.1.3" jsonc-parser "3.3.1" @@ -267,17 +267,17 @@ symbol-observable "4.0.0" yargs "17.7.2" -"@angular/common@18.2.10": - version "18.2.10" - resolved "https://registry.yarnpkg.com/@angular/common/-/common-18.2.10.tgz#7c6196c21c05f0428130436d4dbcd4e63dc3f2b4" - integrity sha512-YzTCmuqLiOuT+Yv07vuKymDWiebOVZ8BuXakJiz4EM7FMoOw5gICHJ04jepZSjDNWpA16e7kJSdt5ucnmvCFDQ== +"@angular/common@18.2.13": + version "18.2.13" + resolved "https://registry.yarnpkg.com/@angular/common/-/common-18.2.13.tgz#cb277651585a6d5aa17c9c1f8e4bbcd85c770517" + integrity sha512-4ZqrNp1PoZo7VNvW+sbSc2CB2axP1sCH2wXl8B0wdjsj8JY1hF1OhuugwhpAHtGxqewed2kCXayE+ZJqSTV4jw== dependencies: tslib "^2.3.0" -"@angular/compiler-cli@18.2.10": - version "18.2.10" - resolved "https://registry.yarnpkg.com/@angular/compiler-cli/-/compiler-cli-18.2.10.tgz#43e8474289942d1128a300c5f74e0ee470dbc943" - integrity sha512-CNFStKWMB89MFKAZZFUOhoQi+fHqRLgNOOrI73LjizXixvngEh3BDZJRtK9hbSGG+giujBrummEA60CWAw69MA== +"@angular/compiler-cli@18.2.13": + version "18.2.13" + resolved "https://registry.yarnpkg.com/@angular/compiler-cli/-/compiler-cli-18.2.13.tgz#3fb9389d9250ce1c09438913094459acfbc18235" + integrity sha512-DBSh4AQwkiJDSiVvJATRmjxf6wyUs9pwQLgaFdSlfuTRO+sdb0J2z1r3BYm8t0IqdoyXzdZq2YCH43EmyvD71g== dependencies: "@babel/core" "7.25.2" "@jridgewell/sourcemap-codec" "^1.4.14" @@ -288,17 +288,17 @@ tslib "^2.3.0" yargs "^17.2.1" -"@angular/compiler@18.2.10": - version "18.2.10" - resolved "https://registry.yarnpkg.com/@angular/compiler/-/compiler-18.2.10.tgz#8c75411d651a7dbc836f68242da6dea8da3f3d56" - integrity sha512-cu+Uq1nnyl00Glg0+2uvm+Xpaq5b4YvWpaLGGtit7uGETAJ4L/frlBVeaTRhEoaIAGBI+RRlyuFLae+etQDA0w== +"@angular/compiler@18.2.13": + version "18.2.13" + resolved "https://registry.yarnpkg.com/@angular/compiler/-/compiler-18.2.13.tgz#cf5d67254bd04a49ecfce0d2d6dc3d9f2234de3c" + integrity sha512-TzWcrkopyjFF+WeDr2cRe8CcHjU72KfYV3Sm2TkBkcXrkYX5sDjGWrBGrG3hRB4e4okqchrOCvm1MiTdy2vKMA== dependencies: tslib "^2.3.0" -"@angular/core@18.2.10": - version "18.2.10" - resolved "https://registry.yarnpkg.com/@angular/core/-/core-18.2.10.tgz#1de4fa5dd998c5d09acf0dc600abecba935d321f" - integrity sha512-EfxVz0pLaxnOppOYkdhnaUkk8HZT+uxaAGpJD3ppAa7YAWTE9xIGoNJmtS33cZNNOnvriMkdv7yn6pDtV4ct+Q== +"@angular/core@18.2.13": + version "18.2.13" + resolved "https://registry.yarnpkg.com/@angular/core/-/core-18.2.13.tgz#50a4269386201b31105dffd6e14c84ca29859cd9" + integrity sha512-8mbWHMgO95OuFV1Ejy4oKmbe9NOJ3WazQf/f7wks8Bck7pcihd0IKhlPBNjFllbF5o+04EYSwFhEtvEgjMDClA== dependencies: tslib "^2.3.0" @@ -309,43 +309,43 @@ dependencies: tslib "^2.3.0" -"@angular/forms@18.2.10": - version "18.2.10" - resolved "https://registry.yarnpkg.com/@angular/forms/-/forms-18.2.10.tgz#86eaefa373f0e9f571547bd2862b2a94de94fa43" - integrity sha512-2VprGB+enJIeqfz2oALmP/G/UiFzpZW6PHgyZXhk/0J/UMsa26JoYxwDFvfdm/WGTrB+VaQEG7in5xwiFPAFtQ== +"@angular/forms@18.2.13": + version "18.2.13" + resolved "https://registry.yarnpkg.com/@angular/forms/-/forms-18.2.13.tgz#119f9b32b0da5e2f1bc3c07f506a645e5635f583" + integrity sha512-A67D867fu3DSBhdLWWZl/F5pr7v2+dRM2u3U7ZJ0ewh4a+sv+0yqWdJW+a8xIoiHxS+btGEJL2qAKJiH+MCFfg== dependencies: tslib "^2.3.0" -"@angular/language-service@18.2.10": - version "18.2.10" - resolved "https://registry.yarnpkg.com/@angular/language-service/-/language-service-18.2.10.tgz#26222ade9203b3fbcb18751762fdce351b2ca205" - integrity sha512-V5yslbkFrBhGr70yK7Bauk8186Di9W8OQtR41fEvPEHecQ5IWjnXzPabl0rjOPtpcAARAYnbIq3sqjQygiX6fw== +"@angular/language-service@18.2.13": + version "18.2.13" + resolved "https://registry.yarnpkg.com/@angular/language-service/-/language-service-18.2.13.tgz#dd5555f7a74ffd5ba361e5bb615d6ea1e1719f93" + integrity sha512-4E4VJDrbOAxS69F9C1twQPbR9AjY47Qlz8+lwg5lJOyUJ4GoEThLbXKfadt/vIeYBwMJ7fIsYWXD0Dlmxh4k+w== -"@angular/material@18.2.11": - version "18.2.11" - resolved "https://registry.yarnpkg.com/@angular/material/-/material-18.2.11.tgz#526a4bc156e326c6249ccfc87e5804079b8b9e49" - integrity sha512-VPfnpwmg6p5DsH1UMfOXjKA+qAbUx6nyinGWpx4+ntr/T1oEhRk5CnoOtVS0Xk0rnRSbEF6ayjDBH2YPR9ol3A== +"@angular/material@18.2.14": + version "18.2.14" + resolved "https://registry.yarnpkg.com/@angular/material/-/material-18.2.14.tgz#2521916b650d16399b8d545d8f4213bc0922b5c6" + integrity sha512-28pxzJP49Mymt664WnCtPkKeg7kXUsQKTKGf/Kl95rNTEdTJLbnlcc8wV0rT0yQNR7kXgpfBnG7h0ETLv/iu5Q== dependencies: tslib "^2.3.0" -"@angular/platform-browser-dynamic@18.2.10": - version "18.2.10" - resolved "https://registry.yarnpkg.com/@angular/platform-browser-dynamic/-/platform-browser-dynamic-18.2.10.tgz#bfbdb68343b0ddf5a46e6ba678d18c2627d46bbd" - integrity sha512-syKyOTgfQnMxfpDRP1khTSPZ5dsMgA8YQwNF6KsB3eZQl15CKSka7bzjMOUWeZ8M3WShOp1AzTf0MfwNeh0UBA== +"@angular/platform-browser-dynamic@18.2.13": + version "18.2.13" + resolved "https://registry.yarnpkg.com/@angular/platform-browser-dynamic/-/platform-browser-dynamic-18.2.13.tgz#6b649f750d2a3d5b0cfaa5fa289f85841f972102" + integrity sha512-kbQCf9+8EpuJC7buBxhSiwBtXvjAwAKh6MznD6zd2pyCYqfY6gfRCZQRtK59IfgVtKmEONWI9grEyNIRoTmqJg== dependencies: tslib "^2.3.0" -"@angular/platform-browser@18.2.10": - version "18.2.10" - resolved "https://registry.yarnpkg.com/@angular/platform-browser/-/platform-browser-18.2.10.tgz#0a3164bb6b4cf7926cfcbfdcfa98abc1f89025b0" - integrity sha512-zKyRKFr3AaEA4SE/DEeb5FWHJutT26avHZog6ZGDkMeMN12zMtSqjPuTSgmDXCWleoOkzbb+nhAQ+fK/EyGyPA== +"@angular/platform-browser@18.2.13": + version "18.2.13" + resolved "https://registry.yarnpkg.com/@angular/platform-browser/-/platform-browser-18.2.13.tgz#59458eb486ec97b2257e7469bb650358492fb823" + integrity sha512-tu7ZzY6qD3ATdWFzcTcsAKe7M6cJeWbT/4/bF9unyGO3XBPcNYDKoiz10+7ap2PUd0fmPwvuvTvSNJiFEBnB8Q== dependencies: tslib "^2.3.0" -"@angular/router@18.2.10": - version "18.2.10" - resolved "https://registry.yarnpkg.com/@angular/router/-/router-18.2.10.tgz#da3e3411d35579e078e8246556a712ce313ed0d2" - integrity sha512-ZqJgOGOfvW0epsc7pIo7DffZqYHo3O9aUCVepZAhOxqtjF/sfhX2fy+A0xopTIiR0eM3LrT823V+2hjlBHj+CA== +"@angular/router@18.2.13": + version "18.2.13" + resolved "https://registry.yarnpkg.com/@angular/router/-/router-18.2.13.tgz#c846ea8ea09a49fea02bda20daca0b46c66df758" + integrity sha512-VKmfgi/r/CkyBq9nChQ/ptmfu0JT/8ONnLVJ5H+SkFLRYJcIRyHLKjRihMCyVm6xM5yktOdCaW73NTQrFz7+bg== dependencies: tslib "^2.3.0" @@ -1619,35 +1619,35 @@ integrity sha512-Arm+WgUFLUATuoxCJcahGuk6Yj9Pzxd6l11Zb/2aAuv5kWWvvfhLFo2fni4uSK5vzlUdCGZ/BdV5tH8klj8p8g== "@eslint-community/eslint-utils@^4.2.0", "@eslint-community/eslint-utils@^4.4.0": - version "4.4.0" - resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59" - integrity sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA== + version "4.4.1" + resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.1.tgz#d1145bf2c20132d6400495d6df4bf59362fd9d56" + integrity sha512-s3O3waFUrMV8P/XaF/+ZTp1X9XBZW1a4B97ZnjQF2KYWaFD2A8KyFBsrsfSjEmjn3RGWAIuvlneuZm3CUK3jbA== dependencies: - eslint-visitor-keys "^3.3.0" + eslint-visitor-keys "^3.4.3" -"@eslint-community/regexpp@^4.10.0", "@eslint-community/regexpp@^4.11.0": - version "4.11.1" - resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.11.1.tgz#a547badfc719eb3e5f4b556325e542fbe9d7a18f" - integrity sha512-m4DVN9ZqskZoLU5GlWZadwDnYo3vAEydiUayB9widCl9ffWx2IvPnp6n3on5rJmziJSw9Bv+Z3ChDVdMwXCY8Q== +"@eslint-community/regexpp@^4.10.0", "@eslint-community/regexpp@^4.12.1": + version "4.12.1" + resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.12.1.tgz#cfc6cffe39df390a3841cde2abccf92eaa7ae0e0" + integrity sha512-CCZCDJuduB9OUkFkY2IgppNZMi2lBQgD2qzwXkEia16cge2pijY/aXi96CJMquDMn3nJdlPV1A5KrJEXwfLNzQ== -"@eslint/config-array@^0.18.0": - version "0.18.0" - resolved "https://registry.yarnpkg.com/@eslint/config-array/-/config-array-0.18.0.tgz#37d8fe656e0d5e3dbaea7758ea56540867fd074d" - integrity sha512-fTxvnS1sRMu3+JjXwJG0j/i4RT9u4qJ+lqS/yCGap4lH4zZGzQ7tu+xZqQmcMZq5OBZDL4QRxQzRjkWcGt8IVw== +"@eslint/config-array@^0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@eslint/config-array/-/config-array-0.19.0.tgz#3251a528998de914d59bb21ba4c11767cf1b3519" + integrity sha512-zdHg2FPIFNKPdcHWtiNT+jEFCHYVplAXRDlQDyqy0zGx/q2parwh7brGJSiTxRk/TSMkbM//zt/f5CHgyTyaSQ== dependencies: "@eslint/object-schema" "^2.1.4" debug "^4.3.1" minimatch "^3.1.2" -"@eslint/core@^0.7.0": - version "0.7.0" - resolved "https://registry.yarnpkg.com/@eslint/core/-/core-0.7.0.tgz#a1bb4b6a4e742a5ff1894b7ee76fbf884ec72bd3" - integrity sha512-xp5Jirz5DyPYlPiKat8jaq0EmYvDXKKpzTbxXMpT9eqlRJkRKIz9AGMdlvYjih+im+QlhWrpvVjl8IPC/lHlUw== +"@eslint/core@^0.9.0": + version "0.9.0" + resolved "https://registry.yarnpkg.com/@eslint/core/-/core-0.9.0.tgz#168ee076f94b152c01ca416c3e5cf82290ab4fcd" + integrity sha512-7ATR9F0e4W85D/0w7cU0SNj7qkAexMG+bAHEZOjo9akvGuhHE2m7umzWzfnpa0XAg5Kxc1BWmtPMV67jJ+9VUg== -"@eslint/eslintrc@^3.1.0": - version "3.1.0" - resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-3.1.0.tgz#dbd3482bfd91efa663cbe7aa1f506839868207b6" - integrity sha512-4Bfj15dVJdoy3RfZmmo86RK1Fwzn6SstsvK9JS+BaVKqC6QQQQyXekNaC+g+LKNgkQ+2VhGAzm6hO40AhMR3zQ== +"@eslint/eslintrc@^3.2.0": + version "3.2.0" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-3.2.0.tgz#57470ac4e2e283a6bf76044d63281196e370542c" + integrity sha512-grOjVNN8P3hjJn/eIETF1wwd12DdnwFDoyceUJLYYdkpbwq3nLi+4fqrTAONx7XDALqlL220wC/RHSC/QTI/0w== dependencies: ajv "^6.12.4" debug "^4.3.2" @@ -1659,20 +1659,20 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@eslint/js@9.13.0": - version "9.13.0" - resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.13.0.tgz#c5f89bcd57eb54d5d4fa8b77693e9c28dc97e547" - integrity sha512-IFLyoY4d72Z5y/6o/BazFBezupzI/taV8sGumxTAVw3lXG9A6md1Dc34T9s1FoD/an9pJH8RHbAxsaEbBed9lA== +"@eslint/js@9.15.0": + version "9.15.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.15.0.tgz#df0e24fe869143b59731942128c19938fdbadfb5" + integrity sha512-tMTqrY+EzbXmKJR5ToI8lxu7jaN5EdmrBFJpQk5JmSlyLsx6o4t27r883K5xsLuCYCpfKBCGswMSWXsM+jB7lg== "@eslint/object-schema@^2.1.4": version "2.1.4" resolved "https://registry.yarnpkg.com/@eslint/object-schema/-/object-schema-2.1.4.tgz#9e69f8bb4031e11df79e03db09f9dbbae1740843" integrity sha512-BsWiH1yFGjXXS2yvrf5LyuoSIIbPrGUWob917o+BTKuZ7qJdxX8aJLRxs1fS9n6r7vESrq1OUqb68dANcFXuQQ== -"@eslint/plugin-kit@^0.2.0": - version "0.2.1" - resolved "https://registry.yarnpkg.com/@eslint/plugin-kit/-/plugin-kit-0.2.1.tgz#cd14fe2db79fa639839dfef4105e83bad1814482" - integrity sha512-HFZ4Mp26nbWk9d/BpvP0YNL6W4UoZF0VFcTw/aPPA8RpOxeFQgK+ClABGgAUXs9Y/RGX/l1vOmrqz1MQt9MNuw== +"@eslint/plugin-kit@^0.2.3": + version "0.2.3" + resolved "https://registry.yarnpkg.com/@eslint/plugin-kit/-/plugin-kit-0.2.3.tgz#812980a6a41ecf3a8341719f92a6d1e784a2e0e8" + integrity sha512-2b/g5hRmpbb1o4GnTZax9N9m0FXzz9OV42ZzI4rDDMDuHUqigAiQCEWChBWCY4ztAGVRjoWT19v0yMmc5/L5kA== dependencies: levn "^0.4.1" @@ -1715,18 +1715,6 @@ dependencies: tslib "^2.3.0" -"@fortawesome/fontawesome-common-types@6.6.0": - version "6.6.0" - resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.6.0.tgz#31ab07ca6a06358c5de4d295d4711b675006163f" - integrity sha512-xyX0X9mc0kyz9plIyryrRbl7ngsA9jz77mCZJsUkLl+ZKs0KWObgaEBoSgQiYWAsSmjz/yjl0F++Got0Mdp4Rw== - -"@fortawesome/fontawesome-svg-core@^6.6.0": - version "6.6.0" - resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-svg-core/-/fontawesome-svg-core-6.6.0.tgz#2a24c32ef92136e98eae2ff334a27145188295ff" - integrity sha512-KHwPkCk6oRT4HADE7smhfsKudt9N/9lm6EJ5BVg0tD1yPA5hht837fB87F8pn15D8JfTqQOjhKTktwmLMiD7Kg== - dependencies: - "@fortawesome/fontawesome-common-types" "6.6.0" - "@geoman-io/leaflet-geoman-free@2.17.0": version "2.17.0" resolved "https://registry.yarnpkg.com/@geoman-io/leaflet-geoman-free/-/leaflet-geoman-free-2.17.0.tgz#9c8fce5c7a85e5d7ece3a7e5d0b82dbf2329bd81" @@ -1739,17 +1727,17 @@ lodash "4.17.21" polyclip-ts "^0.16.5" -"@humanfs/core@^0.19.0": - version "0.19.0" - resolved "https://registry.yarnpkg.com/@humanfs/core/-/core-0.19.0.tgz#08db7a8c73bb07673d9ebd925f2dad746411fcec" - integrity sha512-2cbWIHbZVEweE853g8jymffCA+NCMiuqeECeBBLm8dg2oFdjuGJhgN4UAbI+6v0CKbbhvtXA4qV8YR5Ji86nmw== +"@humanfs/core@^0.19.1": + version "0.19.1" + resolved "https://registry.yarnpkg.com/@humanfs/core/-/core-0.19.1.tgz#17c55ca7d426733fe3c561906b8173c336b40a77" + integrity sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA== -"@humanfs/node@^0.16.5": - version "0.16.5" - resolved "https://registry.yarnpkg.com/@humanfs/node/-/node-0.16.5.tgz#a9febb7e7ad2aff65890fdc630938f8d20aa84ba" - integrity sha512-KSPA4umqSG4LHYRodq31VDwKAvaTF4xmVlzM8Aeh4PlU1JQ3IG0wiA8C25d3RQ9nJyM3mBHyI53K06VVL/oFFg== +"@humanfs/node@^0.16.6": + version "0.16.6" + resolved "https://registry.yarnpkg.com/@humanfs/node/-/node-0.16.6.tgz#ee2a10eaabd1131987bf0488fd9b820174cd765e" + integrity sha512-YuI2ZHQL78Q5HbhDiBA1X4LmYdXCKCMQIfw0pw7piHJwyREFebJUvrQN4cMssyES6x+vfUbx1CIpaQUKYdQZOw== dependencies: - "@humanfs/core" "^0.19.0" + "@humanfs/core" "^0.19.1" "@humanwhocodes/retry" "^0.3.0" "@humanwhocodes/module-importer@^1.0.1": @@ -1757,11 +1745,16 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz#af5b2691a22b44be847b0ca81641c5fb6ad0172c" integrity sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA== -"@humanwhocodes/retry@^0.3.0", "@humanwhocodes/retry@^0.3.1": +"@humanwhocodes/retry@^0.3.0": version "0.3.1" resolved "https://registry.yarnpkg.com/@humanwhocodes/retry/-/retry-0.3.1.tgz#c72a5c76a9fbaf3488e231b13dc52c0da7bab42a" integrity sha512-JBxkERygn7Bv/GbN5Rv8Ul6LVknS+5Bp6RgDC/O8gEBU/yeH5Ui5C/OlWrTb6qct7LjjfT6Re2NxB0ln0yYybA== +"@humanwhocodes/retry@^0.4.1": + version "0.4.1" + resolved "https://registry.yarnpkg.com/@humanwhocodes/retry/-/retry-0.4.1.tgz#9a96ce501bc62df46c4031fbd970e3cc6b10f07b" + integrity sha512-c7hNEllBlenFTHBky65mhq8WD2kbN9Q6gk0bTk8lSBvc554jpXSkST1iePudpt7+A/AQvuHs9EMqjHDXMY1lrA== + "@iconify/types@^2.0.0": version "2.0.0" resolved "https://registry.yarnpkg.com/@iconify/types/-/types-2.0.0.tgz#ab0e9ea681d6c8a1214f30cd741fe3a20cc57f57" @@ -2360,10 +2353,10 @@ dependencies: tslib "^2.0.0" -"@ngtools/webpack@18.2.11": - version "18.2.11" - resolved "https://registry.yarnpkg.com/@ngtools/webpack/-/webpack-18.2.11.tgz#f9c04e01050e4c601178ed62d1a83ca07908a39d" - integrity sha512-iTdUGJ5O7yMm1DyCzyoMDMxBJ68emUSSXPWbQzEEdcqmtifRebn+VAq4vHN8OmtGM1mtuKeLEsbiZP8ywrw7Ug== +"@ngtools/webpack@18.2.12": + version "18.2.12" + resolved "https://registry.yarnpkg.com/@ngtools/webpack/-/webpack-18.2.12.tgz#876a09f2b3acb0d09d93e906a9ba330d92703568" + integrity sha512-FFJAwtWbtpncMOVNuULPBwFJB7GSjiUwO93eGTzRp8O4EPQ8lCQeFbezQm/NP34+T0+GBLGzPSuQT+muob8YKw== "@ngx-translate/core@^15.0.0": version "15.0.0" @@ -2600,13 +2593,13 @@ resolved "https://registry.yarnpkg.com/@rtsao/scc/-/scc-1.1.0.tgz#927dd2fae9bc3361403ac2c7a00c32ddce9ad7e8" integrity sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g== -"@schematics/angular@18.2.11": - version "18.2.11" - resolved "https://registry.yarnpkg.com/@schematics/angular/-/angular-18.2.11.tgz#861cc5c52712df2c824a598e2932f13dcdfa722f" - integrity sha512-jT54mc9+hPOwie9bji/g2krVuK1kkNh2PNFGwfgCg3Ofmt3hcyOBai1DKuot5uLTX4VCCbvfwiVR/hJniQl2SA== +"@schematics/angular@18.2.12": + version "18.2.12" + resolved "https://registry.yarnpkg.com/@schematics/angular/-/angular-18.2.12.tgz#28df36bcd34e85a33dbd0d39932d5ca2de2a9e52" + integrity sha512-sIoeipsisK5eTLW3XuNZYcal83AfslBbgI7LnV+3VrXwpasKPGHwo2ZdwhCd2IXAkuJ02Iyu7MyV0aQRM9i/3g== dependencies: - "@angular-devkit/core" "18.2.11" - "@angular-devkit/schematics" "18.2.11" + "@angular-devkit/core" "18.2.12" + "@angular-devkit/schematics" "18.2.12" jsonc-parser "3.3.1" "@sigstore/bundle@^2.3.2": @@ -3025,7 +3018,7 @@ dependencies: "@types/node" "*" -"@types/jquery@*", "@types/jquery@^3.5.29", "@types/jquery@^3.5.31": +"@types/jquery@*", "@types/jquery@^3.5.29", "@types/jquery@^3.5.32": version "3.5.32" resolved "https://registry.yarnpkg.com/@types/jquery/-/jquery-3.5.32.tgz#3eb0da20611b92c7c49ebed6163b52a4fdc57def" integrity sha512-b9Xbf4CkMqS02YH8zACqN1xzdxc3cO735Qe5AbSUFmyOiaWAbcpqh9Wna+Uk0vgACvoQHpWDg2rGdHkYPLmCiQ== @@ -3068,21 +3061,21 @@ dependencies: "@types/leaflet" "*" -"@types/leaflet.markercluster@1.5.4": - version "1.5.4" - resolved "https://registry.yarnpkg.com/@types/leaflet.markercluster/-/leaflet.markercluster-1.5.4.tgz#2ab43417cf3f6a42d0f1baf4e1c8f659cf1dc3a1" - integrity sha512-tfMP8J62+wfsVLDLGh5Zh1JZxijCaBmVsMAX78MkLPwvPitmZZtSin5aWOVRhZrCS+pEOZwNzexbfWXlY+7yjg== +"@types/leaflet.markercluster@1.5.5": + version "1.5.5" + resolved "https://registry.yarnpkg.com/@types/leaflet.markercluster/-/leaflet.markercluster-1.5.5.tgz#e420caf5b9f6300ab4869fc877d08a9eaf200eec" + integrity sha512-TkWOhSHDM1ANxmLi+uK0PjsVcjIKBr8CLV2WoF16dIdeFmC0Cj5P5axkI3C1Xsi4+ht6EU8+BfEbbqEF9icPrg== dependencies: "@types/leaflet" "*" -"@types/leaflet@*", "@types/leaflet@1.9.12": - version "1.9.12" - resolved "https://registry.yarnpkg.com/@types/leaflet/-/leaflet-1.9.12.tgz#a6626a0b3fba36fd34723d6e95b22e8024781ad6" - integrity sha512-BK7XS+NyRI291HIo0HCfE18Lp8oA30H1gpi1tf0mF3TgiCEzanQjOqNZ4x126SXzzi2oNSZhZ5axJp1k0iM6jg== +"@types/leaflet@*", "@types/leaflet@1.9.14": + version "1.9.14" + resolved "https://registry.yarnpkg.com/@types/leaflet/-/leaflet-1.9.14.tgz#f008fef281a05e457bbae9f00a82c836af1b9b03" + integrity sha512-sx2q6MDJaajwhKeVgPSvqXd8rhNJSTA3tMidQGduZn9S6WBYxDkCpSpV5xXEmSg7Cgdk/5vJGhVF1kMYLzauBg== dependencies: "@types/geojson" "*" -"@types/lodash@^4.17.10": +"@types/lodash@^4.17.13": version "4.17.13" resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.17.13.tgz#786e2d67cfd95e32862143abe7463a7f90c300eb" integrity sha512-lfx+dftrEZcdBPczf9d0Qv0x+j/rfNCMuC6OcfXmO8gkfeNAY88PgKUbvG56whcN23gc27yenwF6oJZXGFpYxg== @@ -3118,10 +3111,12 @@ dependencies: undici-types "~6.19.8" -"@types/node@~18.15.11": - version "18.15.13" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.15.13.tgz#f64277c341150c979e42b00e4ac289290c9df469" - integrity sha512-N+0kuo9KgrUQ1Sn/ifDXsvg0TTleP7rIy4zOBGECxAljqvqfqpTfzx0Q1NUedOixRMBfe2Whhb056a42cWs26Q== +"@types/node@~20.17.8": + version "20.17.8" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.17.8.tgz#42748cdb169adf5be7c9760604c72820c7b7d560" + integrity sha512-ahz2g6/oqbKalW9sPv6L2iRbhLnojxjYWspAqhjvqSWBgGebEJT5GvRmk0QXPj3sbC6rU0GTQjPLQkmR8CObvA== + dependencies: + undici-types "~6.19.2" "@types/parse-json@^4.0.0": version "4.0.2" @@ -3240,62 +3235,62 @@ dependencies: "@types/node" "*" -"@typescript-eslint/eslint-plugin@^8.12.2": - version "8.12.2" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.12.2.tgz#c2ef660bb83fd1432368319312a2581fc92ccac1" - integrity sha512-gQxbxM8mcxBwaEmWdtLCIGLfixBMHhQjBqR8sVWNTPpcj45WlYL2IObS/DNMLH1DBP0n8qz+aiiLTGfopPEebw== +"@typescript-eslint/eslint-plugin@^8.16.0": + version "8.16.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.16.0.tgz#ac56825bcdf3b392fc76a94b1315d4a162f201a6" + integrity sha512-5YTHKV8MYlyMI6BaEG7crQ9BhSc8RxzshOReKwZwRWN0+XvvTOm+L/UYLCYxFpfwYuAAqhxiq4yae0CMFwbL7Q== dependencies: "@eslint-community/regexpp" "^4.10.0" - "@typescript-eslint/scope-manager" "8.12.2" - "@typescript-eslint/type-utils" "8.12.2" - "@typescript-eslint/utils" "8.12.2" - "@typescript-eslint/visitor-keys" "8.12.2" + "@typescript-eslint/scope-manager" "8.16.0" + "@typescript-eslint/type-utils" "8.16.0" + "@typescript-eslint/utils" "8.16.0" + "@typescript-eslint/visitor-keys" "8.16.0" graphemer "^1.4.0" ignore "^5.3.1" natural-compare "^1.4.0" ts-api-utils "^1.3.0" -"@typescript-eslint/parser@^8.12.2": - version "8.12.2" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-8.12.2.tgz#2e8173b34e1685e918b2d571c16c906d3747bad2" - integrity sha512-MrvlXNfGPLH3Z+r7Tk+Z5moZAc0dzdVjTgUgwsdGweH7lydysQsnSww3nAmsq8blFuRD5VRlAr9YdEFw3e6PBw== +"@typescript-eslint/parser@^8.16.0": + version "8.16.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-8.16.0.tgz#ee5b2d6241c1ab3e2e53f03fd5a32d8e266d8e06" + integrity sha512-D7DbgGFtsqIPIFMPJwCad9Gfi/hC0PWErRRHFnaCWoEDYi5tQUDiJCTmGUbBiLzjqAck4KcXt9Ayj0CNlIrF+w== dependencies: - "@typescript-eslint/scope-manager" "8.12.2" - "@typescript-eslint/types" "8.12.2" - "@typescript-eslint/typescript-estree" "8.12.2" - "@typescript-eslint/visitor-keys" "8.12.2" + "@typescript-eslint/scope-manager" "8.16.0" + "@typescript-eslint/types" "8.16.0" + "@typescript-eslint/typescript-estree" "8.16.0" + "@typescript-eslint/visitor-keys" "8.16.0" debug "^4.3.4" -"@typescript-eslint/scope-manager@8.12.2": - version "8.12.2" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-8.12.2.tgz#6db0213745e6392c8e90fe9af5915e6da32eb94a" - integrity sha512-gPLpLtrj9aMHOvxJkSbDBmbRuYdtiEbnvO25bCMza3DhMjTQw0u7Y1M+YR5JPbMsXXnSPuCf5hfq0nEkQDL/JQ== +"@typescript-eslint/scope-manager@8.16.0": + version "8.16.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-8.16.0.tgz#ebc9a3b399a69a6052f3d88174456dd399ef5905" + integrity sha512-mwsZWubQvBki2t5565uxF0EYvG+FwdFb8bMtDuGQLdCCnGPrDEDvm1gtfynuKlnpzeBRqdFCkMf9jg1fnAK8sg== dependencies: - "@typescript-eslint/types" "8.12.2" - "@typescript-eslint/visitor-keys" "8.12.2" + "@typescript-eslint/types" "8.16.0" + "@typescript-eslint/visitor-keys" "8.16.0" -"@typescript-eslint/type-utils@8.12.2": - version "8.12.2" - resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-8.12.2.tgz#132b0c52d45f6814e6f2e32416c7951ed480b016" - integrity sha512-bwuU4TAogPI+1q/IJSKuD4shBLc/d2vGcRT588q+jzayQyjVK2X6v/fbR4InY2U2sgf8MEvVCqEWUzYzgBNcGQ== +"@typescript-eslint/type-utils@8.16.0": + version "8.16.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-8.16.0.tgz#585388735f7ac390f07c885845c3d185d1b64740" + integrity sha512-IqZHGG+g1XCWX9NyqnI/0CX5LL8/18awQqmkZSl2ynn8F76j579dByc0jhfVSnSnhf7zv76mKBQv9HQFKvDCgg== dependencies: - "@typescript-eslint/typescript-estree" "8.12.2" - "@typescript-eslint/utils" "8.12.2" + "@typescript-eslint/typescript-estree" "8.16.0" + "@typescript-eslint/utils" "8.16.0" debug "^4.3.4" ts-api-utils "^1.3.0" -"@typescript-eslint/types@8.12.2", "@typescript-eslint/types@^8.12.2": - version "8.12.2" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-8.12.2.tgz#8d70098c0e90442495b53d0296acdca6d0f3f73c" - integrity sha512-VwDwMF1SZ7wPBUZwmMdnDJ6sIFk4K4s+ALKLP6aIQsISkPv8jhiw65sAK6SuWODN/ix+m+HgbYDkH+zLjrzvOA== +"@typescript-eslint/types@8.16.0", "@typescript-eslint/types@^8.16.0": + version "8.16.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-8.16.0.tgz#49c92ae1b57942458ab83d9ec7ccab3005e64737" + integrity sha512-NzrHj6thBAOSE4d9bsuRNMvk+BvaQvmY4dDglgkgGC0EW/tB3Kelnp3tAKH87GEwzoxgeQn9fNGRyFJM/xd+GQ== -"@typescript-eslint/typescript-estree@8.12.2": - version "8.12.2" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-8.12.2.tgz#206df9b1cbff212aaa9401985ef99f04daa84da5" - integrity sha512-mME5MDwGe30Pq9zKPvyduyU86PH7aixwqYR2grTglAdB+AN8xXQ1vFGpYaUSJ5o5P/5znsSBeNcs5g5/2aQwow== +"@typescript-eslint/typescript-estree@8.16.0": + version "8.16.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-8.16.0.tgz#9d741e56e5b13469b5190e763432ce5551a9300c" + integrity sha512-E2+9IzzXMc1iaBy9zmo+UYvluE3TW7bCGWSF41hVWUE01o8nzr1rvOQYSxelxr6StUvRcTMe633eY8mXASMaNw== dependencies: - "@typescript-eslint/types" "8.12.2" - "@typescript-eslint/visitor-keys" "8.12.2" + "@typescript-eslint/types" "8.16.0" + "@typescript-eslint/visitor-keys" "8.16.0" debug "^4.3.4" fast-glob "^3.3.2" is-glob "^4.0.3" @@ -3303,23 +3298,23 @@ semver "^7.6.0" ts-api-utils "^1.3.0" -"@typescript-eslint/utils@8.12.2", "@typescript-eslint/utils@^8.12.2": - version "8.12.2" - resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-8.12.2.tgz#726cc9f49f5866605bd15bbc1768ffc15637930e" - integrity sha512-UTTuDIX3fkfAz6iSVa5rTuSfWIYZ6ATtEocQ/umkRSyC9O919lbZ8dcH7mysshrCdrAM03skJOEYaBugxN+M6A== +"@typescript-eslint/utils@8.16.0", "@typescript-eslint/utils@^8.16.0": + version "8.16.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-8.16.0.tgz#c71264c437157feaa97842809836254a6fc833c3" + integrity sha512-C1zRy/mOL8Pj157GiX4kaw7iyRLKfJXBR3L82hk5kS/GyHcOFmy4YUq/zfZti72I9wnuQtA/+xzft4wCC8PJdA== dependencies: "@eslint-community/eslint-utils" "^4.4.0" - "@typescript-eslint/scope-manager" "8.12.2" - "@typescript-eslint/types" "8.12.2" - "@typescript-eslint/typescript-estree" "8.12.2" + "@typescript-eslint/scope-manager" "8.16.0" + "@typescript-eslint/types" "8.16.0" + "@typescript-eslint/typescript-estree" "8.16.0" -"@typescript-eslint/visitor-keys@8.12.2": - version "8.12.2" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-8.12.2.tgz#94d7410f78eb6d134b9fcabaf1eeedb910ba8c38" - integrity sha512-PChz8UaKQAVNHghsHcPyx1OMHoFRUEA7rJSK/mDhdq85bk+PLsUHUBqTQTFt18VJZbmxBovM65fezlheQRsSDA== +"@typescript-eslint/visitor-keys@8.16.0": + version "8.16.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-8.16.0.tgz#d5086afc060b01ff7a4ecab8d49d13d5a7b07705" + integrity sha512-pq19gbaMOmFE3CbL0ZB8J8BFCo2ckfHBfaIsaOZgBIF4EoISJIdLX5xRhd0FGB0LlHReNRuzoJoMGpTjq8F2CQ== dependencies: - "@typescript-eslint/types" "8.12.2" - eslint-visitor-keys "^3.4.3" + "@typescript-eslint/types" "8.16.0" + eslint-visitor-keys "^4.2.0" "@vitejs/plugin-basic-ssl@1.1.0": version "1.1.0" @@ -3475,10 +3470,10 @@ accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.8: mime-types "~2.1.34" negotiator "0.6.3" -ace-builds@1.36.2, ace-builds@^1.32.8: - version "1.36.2" - resolved "https://registry.yarnpkg.com/ace-builds/-/ace-builds-1.36.2.tgz#9499bd59e839a335ac4850e74549ca8d849dc554" - integrity sha512-eqqfbGwx/GKjM/EnFu4QtQ+d2NNBu84MGgxoG8R5iyFpcVeQ4p9YlTL+ZzdEJqhdkASqoqOxCSNNGyB6lvMm+A== +ace-builds@1.36.5, ace-builds@^1.32.8: + version "1.36.5" + resolved "https://registry.yarnpkg.com/ace-builds/-/ace-builds-1.36.5.tgz#ae9cc7a32eccc2f484926131c00545cd6b78a6a6" + integrity sha512-mZ5KVanRT6nLRDLqtG/1YQQLX/gZVC/v526cm1Ru/MTSlrbweSmqv2ZT0d2GaHpJq035MwCMIrj+LgDAUnDXrg== ace-diff@^3.0.3: version "3.0.3" @@ -4097,7 +4092,7 @@ chevrotain@~11.0.3: "@chevrotain/utils" "11.0.3" lodash-es "4.17.21" -"chokidar@>=3.0.0 <4.0.0", chokidar@^3.5.3, chokidar@^3.6.0: +"chokidar@>=3.0.0 <4.0.0", chokidar@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.6.0.tgz#197c6cc669ef2a8dc5e7b4d97ee4e092c3eb0d5b" integrity sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw== @@ -4414,10 +4409,10 @@ core-js-compat@^3.37.1, core-js-compat@^3.38.0: dependencies: browserslist "^4.23.3" -core-js@^3.38.1: - version "3.38.1" - resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.38.1.tgz#aa375b79a286a670388a1a363363d53677c0383e" - integrity sha512-OP35aUorbU3Zvlx7pjsFdu1rGNnD4pgw/CWoYzRY3t2EzoVT7shKHY1dlAy3f41cGIO7ZDPQimhGFTlEYkG/Hw== +core-js@^3.39.0: + version "3.39.0" + resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.39.0.tgz#57f7647f4d2d030c32a72ea23a0555b2eaa30f83" + integrity sha512-raM0ew0/jJUqkJ0E6e8UDtl+y/7ktFivgWvqw8dNSQeNWoSDLvQ1H/RN3aPXB9tBd4/FhyR4RDPGhsNIMsAn7g== core-util-is@~1.0.0: version "1.0.3" @@ -4488,10 +4483,10 @@ critters@0.0.24: postcss "^8.4.23" postcss-media-query-parser "^0.2.3" -cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: - version "7.0.3" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" - integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== +cross-spawn@^7.0.0, cross-spawn@^7.0.3, cross-spawn@^7.0.5: + version "7.0.6" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f" + integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA== dependencies: path-key "^3.1.0" shebang-command "^2.0.0" @@ -5423,10 +5418,10 @@ eslint-plugin-import@latest: string.prototype.trimend "^1.0.8" tsconfig-paths "^3.15.0" -eslint-plugin-jsdoc@^50.4.3: - version "50.4.3" - resolved "https://registry.yarnpkg.com/eslint-plugin-jsdoc/-/eslint-plugin-jsdoc-50.4.3.tgz#38adf595555933775943771e906422b25cdfc780" - integrity sha512-uWtwFxGRv6B8sU63HZM5dAGDhgsatb+LONwmILZJhdRALLOkCX2HFZhdL/Kw2ls8SQMAVEfK+LmnEfxInRN8HA== +eslint-plugin-jsdoc@^50.6.0: + version "50.6.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-jsdoc/-/eslint-plugin-jsdoc-50.6.0.tgz#2c6049a40305313174a30212bc360e775b797a0a" + integrity sha512-tCNp4fR79Le3dYTPB0dKEv7yFyvGkUCa+Z3yuTrrNGGOxBlXo9Pn0PEgroOZikUQOGjxoGMVKNjrOHcYEdfszg== dependencies: "@es-joy/jsdoccomment" "~0.49.0" are-docs-informative "^0.0.2" @@ -5461,49 +5456,49 @@ eslint-scope@5.1.1: esrecurse "^4.3.0" estraverse "^4.1.1" -eslint-scope@^8.0.2, eslint-scope@^8.1.0: - version "8.1.0" - resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-8.1.0.tgz#70214a174d4cbffbc3e8a26911d8bf51b9ae9d30" - integrity sha512-14dSvlhaVhKKsa9Fx1l8A17s7ah7Ef7wCakJ10LYk6+GYmP9yDti2oq2SEwcyndt6knfcZyhyxwY3i9yL78EQw== +eslint-scope@^8.0.2, eslint-scope@^8.2.0: + version "8.2.0" + resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-8.2.0.tgz#377aa6f1cb5dc7592cfd0b7f892fd0cf352ce442" + integrity sha512-PHlWUfG6lvPc3yvP5A4PNyBL1W8fkDUccmI21JUu/+GKZBoH/W5u6usENXUrWFRsyoW5ACUjFGgAFQp5gUlb/A== dependencies: esrecurse "^4.3.0" estraverse "^5.2.0" -eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.3: +eslint-visitor-keys@^3.4.3: version "3.4.3" resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz#0cd72fe8550e3c2eae156a96a4dddcd1c8ac5800" integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag== -eslint-visitor-keys@^4.1.0, eslint-visitor-keys@^4.2.0: +eslint-visitor-keys@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-4.2.0.tgz#687bacb2af884fcdda8a6e7d65c606f46a14cd45" integrity sha512-UyLnSehNt62FFhSwjZlHmeokpRK59rcz29j+F1/aDgbkbRTk7wIc9XzdoasMUbRNKDM0qQt/+BJ4BrpFeABemw== -eslint@~9.13.0: - version "9.13.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.13.0.tgz#7659014b7dda1ff876ecbd990f726e11c61596e6" - integrity sha512-EYZK6SX6zjFHST/HRytOdA/zE72Cq/bfw45LSyuwrdvcclb/gqV8RRQxywOBEWO2+WDpva6UZa4CcDeJKzUCFA== +eslint@~9.15.0: + version "9.15.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.15.0.tgz#77c684a4e980e82135ebff8ee8f0a9106ce6b8a6" + integrity sha512-7CrWySmIibCgT1Os28lUU6upBshZ+GxybLOrmRzi08kS8MBuO8QA7pXEgYgY5W8vK3e74xv0lpjo9DbaGU9Rkw== dependencies: "@eslint-community/eslint-utils" "^4.2.0" - "@eslint-community/regexpp" "^4.11.0" - "@eslint/config-array" "^0.18.0" - "@eslint/core" "^0.7.0" - "@eslint/eslintrc" "^3.1.0" - "@eslint/js" "9.13.0" - "@eslint/plugin-kit" "^0.2.0" - "@humanfs/node" "^0.16.5" + "@eslint-community/regexpp" "^4.12.1" + "@eslint/config-array" "^0.19.0" + "@eslint/core" "^0.9.0" + "@eslint/eslintrc" "^3.2.0" + "@eslint/js" "9.15.0" + "@eslint/plugin-kit" "^0.2.3" + "@humanfs/node" "^0.16.6" "@humanwhocodes/module-importer" "^1.0.1" - "@humanwhocodes/retry" "^0.3.1" + "@humanwhocodes/retry" "^0.4.1" "@types/estree" "^1.0.6" "@types/json-schema" "^7.0.15" ajv "^6.12.4" chalk "^4.0.0" - cross-spawn "^7.0.2" + cross-spawn "^7.0.5" debug "^4.3.2" escape-string-regexp "^4.0.0" - eslint-scope "^8.1.0" - eslint-visitor-keys "^4.1.0" - espree "^10.2.0" + eslint-scope "^8.2.0" + eslint-visitor-keys "^4.2.0" + espree "^10.3.0" esquery "^1.5.0" esutils "^2.0.2" fast-deep-equal "^3.1.3" @@ -5518,9 +5513,8 @@ eslint@~9.13.0: minimatch "^3.1.2" natural-compare "^1.4.0" optionator "^0.9.3" - text-table "^0.2.0" -espree@^10.0.1, espree@^10.1.0, espree@^10.2.0: +espree@^10.0.1, espree@^10.1.0, espree@^10.3.0: version "10.3.0" resolved "https://registry.yarnpkg.com/espree/-/espree-10.3.0.tgz#29267cf5b0cb98735b65e64ba07e0ed49d1eed8a" integrity sha512-0QYC8b24HWY8zjRnDTL6RiHfDbAWn63qb4LMj1Z4b076A4une81+z03Kg7l7mn/48PUTqoLptSXez8oknU8Clg== @@ -5654,7 +5648,7 @@ fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== -fast-glob@3.3.2, fast-glob@^3.2.5, fast-glob@^3.3.0, fast-glob@^3.3.2: +fast-glob@3.3.2, fast-glob@^3.2.5, fast-glob@^3.3.2: version "3.3.2" resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.3.2.tgz#a904501e57cfdd2ffcded45e99a54fef55e46129" integrity sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow== @@ -5801,9 +5795,9 @@ flat@^5.0.2: integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ== flatted@^3.2.9: - version "3.3.1" - resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.3.1.tgz#21db470729a6734d4997002f439cb308987f567a" - integrity sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw== + version "3.3.2" + resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.3.2.tgz#adba1448a9841bec72b42c532ea23dbbedef1a27" + integrity sha512-AiwGJM8YcNOaobumgtng+6NHuOqC3A7MixFeDafM3X9cIUM+xUXoS5Vfgf+OihAYe20fxqNM9yPBXJzRtZ/4eA== "flot.curvedlines@https://github.com/MichaelZinsmaier/CurvedLines.git#master": version "1.1.1" @@ -6309,7 +6303,12 @@ ignore-walk@^6.0.4: dependencies: minimatch "^9.0.0" -ignore@5.3.2, ignore@^5.2.0, ignore@^5.2.4, ignore@^5.3.1: +ignore@6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-6.0.2.tgz#77cccb72a55796af1b6d2f9eb14fa326d24f4283" + integrity sha512-InwqeHHN2XpumIkMvpl/DCJVrAHgCsG5+cn1XlnLWGwtZBm8QJfSusItfrwx81CTp5agNZqpKU2J/ccC5nGT4A== + +ignore@^5.2.0, ignore@^5.2.4, ignore@^5.3.1: version "5.3.2" resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.2.tgz#3cd40e729f3643fd87cb04e50bf0eb722bc596f5" integrity sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g== @@ -6704,15 +6703,15 @@ jest-worker@^27.4.5: merge-stream "^2.0.0" supports-color "^8.0.0" -jiti@^1.20.0, jiti@^1.21.0: +jiti@^1.20.0, jiti@^1.21.6: version "1.21.6" resolved "https://registry.yarnpkg.com/jiti/-/jiti-1.21.6.tgz#6c7f7398dd4b3142767f9a168af2f317a428d268" integrity sha512-2yTgeWTWzMWkHu6Jp9NKgePDaYHbntiwvYuuJLbbN9vl7DC9DvXKOB2BC3ZZ92D3cvV/aflH0osDfwpHepQ53w== -jquery.terminal@^2.43.1: - version "2.43.1" - resolved "https://registry.yarnpkg.com/jquery.terminal/-/jquery.terminal-2.43.1.tgz#07f022d8853dd0fd8641f989623a30086b4381bb" - integrity sha512-TcezgCLdtAn0dTzPXK6kdnzJV4k9vcdTVxt7l0teFV+7LGWRCFIhs49U3WeNKU0aereFtlR5N5qrSWBGqPlw0A== +jquery.terminal@^2.44.1: + version "2.44.1" + resolved "https://registry.yarnpkg.com/jquery.terminal/-/jquery.terminal-2.44.1.tgz#9a66b28cbc0ffad086dcc375dad08c93bdeae001" + integrity sha512-uyvTsnmtvz7tHo4gvdPImZiKf2p9pQhAg/qkk9sj8vePP1xOTHjm70WKAkeiY4ZUWgIRDfAkyWlaJ6UZWy9m6w== dependencies: "@jcubic/lily" "^0.3.0" "@types/jquery" "^3.5.29" @@ -7095,10 +7094,10 @@ levn@^0.4.1: prelude-ls "^1.2.1" type-check "~0.4.0" -libphonenumber-js@^1.11.10: - version "1.11.12" - resolved "https://registry.yarnpkg.com/libphonenumber-js/-/libphonenumber-js-1.11.12.tgz#e108a4632048255f30b872bb1abbb3356f290fbb" - integrity sha512-QkJn9/D7zZ1ucvT++TQSvZuSA2xAWeUytU+DiEQwbPKLyrDpvbul2AFs1CGbRAPpSCCk47aRAb5DX5mmcayp4g== +libphonenumber-js@^1.11.15: + version "1.11.15" + resolved "https://registry.yarnpkg.com/libphonenumber-js/-/libphonenumber-js-1.11.15.tgz#0947ba02208cf6c44fdf3b07e097a98b3ec945f4" + integrity sha512-M7+rtYi9l5RvMmHyjyoF3BHHUpXTYdJ0PezZGHNs0GyW1lO+K7jxlXpbdIb7a56h0nqLYdjIw+E+z0ciGaJP7g== license-webpack-plugin@4.0.2: version "4.0.2" @@ -7398,7 +7397,7 @@ methods@~1.1.2: resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w== -micromatch@^4.0.2, micromatch@^4.0.4, micromatch@^4.0.5, micromatch@^4.0.8: +micromatch@^4.0.2, micromatch@^4.0.4, micromatch@^4.0.8: version "4.0.8" resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.8.tgz#d66fa18f3a47076789320b9b1af32bd86d9fa202" integrity sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA== @@ -7637,9 +7636,9 @@ mz@^2.7.0: thenify-all "^1.0.0" nanoid@^3.3.7: - version "3.3.7" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.7.tgz#d0c301a691bc8d54efa0a2226ccf3fe2fd656bd8" - integrity sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g== + version "3.3.8" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.8.tgz#b1be3030bee36aaff18bacb375e5cce521684baf" + integrity sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w== natural-compare@^1.4.0: version "1.4.0" @@ -7723,10 +7722,10 @@ ngx-markdown@^18.1.0: mermaid ">= 10.6.0 < 12.0.0" prismjs "^1.28.0" -ngx-sharebuttons@^15.0.3: - version "15.0.3" - resolved "https://registry.yarnpkg.com/ngx-sharebuttons/-/ngx-sharebuttons-15.0.3.tgz#80b87936c7dfa3492beba754cb9b3b53da2acd75" - integrity sha512-wArI5Pbg1xORycjzE5yIeNiC7akG/Unz7HwjIaWCHCCvyh4RmHOz8xv4AE/O8/0hDVWFNfyATc6byuzBYRrmDw== +ngx-sharebuttons@^15.0.6: + version "15.0.6" + resolved "https://registry.yarnpkg.com/ngx-sharebuttons/-/ngx-sharebuttons-15.0.6.tgz#3347d71ef7aefdcb23d54b1ea453dd0fe8df5b87" + integrity sha512-majxKSfcPhPKNR+aUNn/dZWd7W9loFSuS2/beV8Bz2r2vd3FVjswvkGHPNO0q956x7KNZn18Wu+IUmuuF/lxyw== dependencies: tslib "^2.3.0" @@ -8308,7 +8307,7 @@ pathe@^1.1.2: resolved "https://registry.yarnpkg.com/pathe/-/pathe-1.1.2.tgz#6c4cb47a945692e48a1ddd6e4094d170516437ec" integrity sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ== -picocolors@^1.0.0, picocolors@^1.0.1, picocolors@^1.1.0: +picocolors@^1.0.0, picocolors@^1.0.1, picocolors@^1.1.0, picocolors@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b" integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA== @@ -8408,7 +8407,7 @@ postcss-js@^4.0.1: dependencies: camelcase-css "^2.0.1" -postcss-load-config@^4.0.1: +postcss-load-config@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/postcss-load-config/-/postcss-load-config-4.0.2.tgz#7159dcf626118d33e299f485d6afe4aff7c4a3e3" integrity sha512-bSVhyJGL00wMVoPUzAVAnbEoWyqRxkjv64tUl427SKnPrENtq6hJwUojroMz2VB+Q1edmi4IfrAPpami5VVgMQ== @@ -8458,14 +8457,14 @@ postcss-modules-values@^4.0.0: dependencies: icss-utils "^5.0.0" -postcss-nested@^6.0.1: +postcss-nested@^6.2.0: version "6.2.0" resolved "https://registry.yarnpkg.com/postcss-nested/-/postcss-nested-6.2.0.tgz#4c2d22ab5f20b9cb61e2c5c5915950784d068131" integrity sha512-HQbt28KulC5AJzG+cZtj9kvKB93CFCdLvog1WFLf1D+xmMvPGlBstkpTEZfK5+AN9hfJocyBFCNiqyS48bpgzQ== dependencies: postcss-selector-parser "^6.1.1" -postcss-selector-parser@^6.0.11, postcss-selector-parser@^6.0.2, postcss-selector-parser@^6.0.4, postcss-selector-parser@^6.1.1: +postcss-selector-parser@^6.0.2, postcss-selector-parser@^6.0.4, postcss-selector-parser@^6.1.1, postcss-selector-parser@^6.1.2: version "6.1.2" resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz#27ecb41fb0e3b6ba7a1ec84fff347f734c7929de" integrity sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg== @@ -8487,13 +8486,13 @@ postcss@8.4.41: picocolors "^1.0.1" source-map-js "^1.2.0" -postcss@^8.2.14, postcss@^8.4.23, postcss@^8.4.33, postcss@^8.4.4, postcss@^8.4.43, postcss@^8.4.47: - version "8.4.47" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.47.tgz#5bf6c9a010f3e724c503bf03ef7947dcb0fea365" - integrity sha512-56rxCq7G/XfB4EkXq9Egn5GCqugWvDFjafDOThIdMBsI15iqPqR5r15TfSr1YPYeEI19YeaXMCbY6u88Y76GLQ== +postcss@^8.2.14, postcss@^8.4.23, postcss@^8.4.33, postcss@^8.4.4, postcss@^8.4.43, postcss@^8.4.47, postcss@^8.4.49: + version "8.4.49" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.49.tgz#4ea479048ab059ab3ae61d082190fabfd994fe19" + integrity sha512-OCVPnIObs4N29kxTjzLfUryOkvZEq+pf8jTF0lg8E7uETuWHA+v7j3c/xJmiqpX450191LlmZfUKkXxkTry7nA== dependencies: nanoid "^3.3.7" - picocolors "^1.1.0" + picocolors "^1.1.1" source-map-js "^1.2.1" postinstall-prepare@^2.0.0: @@ -8910,7 +8909,7 @@ resolve-url-loader@5.0.0: postcss "^8.2.14" source-map "0.6.1" -resolve@1.22.8, resolve@^1.1.7, resolve@^1.14.2, resolve@^1.19.0, resolve@^1.22.2, resolve@^1.22.4: +resolve@1.22.8, resolve@^1.1.7, resolve@^1.14.2, resolve@^1.19.0, resolve@^1.22.4, resolve@^1.22.8: version "1.22.8" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== @@ -9632,7 +9631,7 @@ stylis@^4.3.1: resolved "https://registry.yarnpkg.com/stylis/-/stylis-4.3.4.tgz#ca5c6c4a35c4784e4e93a2a24dc4e9fa075250a4" integrity sha512-osIBl6BGUmSfDkyH2mB7EFvCJntXDrLhKjHTRj/rK6xLH0yuPrHULDRQzKokSOD4VoorhtKpfcfW1GAntu8now== -sucrase@^3.32.0: +sucrase@^3.35.0: version "3.35.0" resolved "https://registry.yarnpkg.com/sucrase/-/sucrase-3.35.0.tgz#57f17a3d7e19b36d8995f06679d121be914ae263" integrity sha512-8EbVDiu9iN/nESwxeSxDKe0dunta1GOlHufmSSXxMD2z2/tMZpDMpvXQGsc+ajGo8y2uYUmixaSRUc/QPoQ0GA== @@ -9699,33 +9698,33 @@ table-layout@^1.0.2: typical "^5.2.0" wordwrapjs "^4.0.0" -tailwindcss@^3.4.14: - version "3.4.14" - resolved "https://registry.yarnpkg.com/tailwindcss/-/tailwindcss-3.4.14.tgz#6dd23a7f54ec197b19159e91e3bb1e55e7aa73ac" - integrity sha512-IcSvOcTRcUtQQ7ILQL5quRDg7Xs93PdJEk1ZLbhhvJc7uj/OAhYOnruEiwnGgBvUtaUAJ8/mhSw1o8L2jCiENA== +tailwindcss@^3.4.15: + version "3.4.15" + resolved "https://registry.yarnpkg.com/tailwindcss/-/tailwindcss-3.4.15.tgz#04808bf4bf1424b105047d19e7d4bfab368044a9" + integrity sha512-r4MeXnfBmSOuKUWmXe6h2CcyfzJCEk4F0pptO5jlnYSIViUkVmsawj80N5h2lO3gwcmSb4n3PuN+e+GC1Guylw== dependencies: "@alloc/quick-lru" "^5.2.0" arg "^5.0.2" - chokidar "^3.5.3" + chokidar "^3.6.0" didyoumean "^1.2.2" dlv "^1.1.3" - fast-glob "^3.3.0" + fast-glob "^3.3.2" glob-parent "^6.0.2" is-glob "^4.0.3" - jiti "^1.21.0" + jiti "^1.21.6" lilconfig "^2.1.0" - micromatch "^4.0.5" + micromatch "^4.0.8" normalize-path "^3.0.0" object-hash "^3.0.0" - picocolors "^1.0.0" - postcss "^8.4.23" + picocolors "^1.1.1" + postcss "^8.4.47" postcss-import "^15.1.0" postcss-js "^4.0.1" - postcss-load-config "^4.0.1" - postcss-nested "^6.0.1" - postcss-selector-parser "^6.0.11" - resolve "^1.22.2" - sucrase "^3.32.0" + postcss-load-config "^4.0.2" + postcss-nested "^6.2.0" + postcss-selector-parser "^6.1.2" + resolve "^1.22.8" + sucrase "^3.35.0" tapable@^2.1.1, tapable@^2.2.0, tapable@^2.2.1: version "2.2.1" @@ -9782,11 +9781,6 @@ text-segmentation@^1.0.3: dependencies: utrie "^1.0.2" -text-table@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" - integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw== - thenify-all@^1.0.0: version "1.6.0" resolved "https://registry.yarnpkg.com/thenify-all/-/thenify-all-1.6.0.tgz#1a1918d402d8fc3f98fbf234db0bcc8cc10e9726" @@ -9871,9 +9865,9 @@ tree-kill@1.2.2: integrity sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A== ts-api-utils@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/ts-api-utils/-/ts-api-utils-1.3.0.tgz#4b490e27129f1e8e686b45cc4ab63714dc60eea1" - integrity sha512-UQMIo7pb8WRomKR1/+MFVLTroIvDVtMX3K6OUir8ynLyzB8Jeriont2bTAtmNPa1ekAgN7YPDyf6V+ygrdU+eQ== + version "1.4.2" + resolved "https://registry.yarnpkg.com/ts-api-utils/-/ts-api-utils-1.4.2.tgz#a6a6dff26117ac7965624fc118525971edc6a82a" + integrity sha512-ZF5gQIQa/UmzfvxbHZI3JXN0/Jt+vnAfAviNRAMc491laiK6YCLpCW9ft8oaCRFOTxCZtUTE6XB0ZQAe3olntw== ts-dedent@^2.2.0: version "2.2.0" @@ -10056,7 +10050,7 @@ unbox-primitive@^1.0.2: has-symbols "^1.0.3" which-boxed-primitive "^1.0.2" -undici-types@~6.19.8: +undici-types@~6.19.2, undici-types@~6.19.8: version "6.19.8" resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02" integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw== @@ -10516,9 +10510,9 @@ yaml@^1.10.0: integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== yaml@^2.2.2, yaml@^2.3.4: - version "2.6.0" - resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.6.0.tgz#14059ad9d0b1680d0f04d3a60fe00f3a857303c3" - integrity sha512-a6ae//JvKDEra2kdi1qzCyrJW/WZCgFi8ydDV+eXExl95t+5R+ijnqHJbz9tmMh8FUjx3iv2fCQ4dclAQlO2UQ== + version "2.6.1" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.6.1.tgz#42f2b1ba89203f374609572d5349fb8686500773" + integrity sha512-7r0XPzioN/Q9kXBro/XPnA6kznR73DHq+GXh5ON7ZozRO6aMjbmiBuKste2wslTFkC5d1dw0GooOCepZXJ2SAg== yargs-parser@^18.1.2: version "18.1.3" From 65f94878d24f17842602a98b27d247e7aad75759 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 27 Nov 2024 13:36:48 +0200 Subject: [PATCH 51/71] removed using TTL from tenant profile --- .../TbSaveToCustomCassandraTableNode.java | 24 +++------- ...CustomCassandraTableNodeConfiguration.java | 4 +- .../engine/telemetry/TbMsgTimeseriesNode.java | 2 +- .../TbSaveToCustomCassandraTableNodeTest.java | 45 ++++--------------- 4 files changed, 17 insertions(+), 58 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index ddc4357674..7d4cd64179 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -38,10 +38,8 @@ import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; -import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.rule.RuleChainType; -import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.dao.cassandra.CassandraCluster; @@ -54,7 +52,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import static org.thingsboard.common.util.DonAsynchron.withCallback; @@ -88,7 +85,6 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { private PreparedStatement saveStmt; private ExecutorService readResultsProcessingExecutor; private Map fieldsMap; - private Integer ttl; @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { @@ -100,20 +96,10 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { if (!isTableExists()) { throw new TbNodeException("Table '" + TABLE_PREFIX + config.getTableName() + "' does not exist in Cassandra cluster."); } - ctx.addTenantProfileListener(this::onTenantProfileUpdate); - onTenantProfileUpdate(ctx.getTenantProfile()); startExecutor(); saveStmt = getSaveStmt(); } - private void onTenantProfileUpdate(TenantProfile tenantProfile) { - DefaultTenantProfileConfiguration configuration = (DefaultTenantProfileConfiguration) tenantProfile.getProfileData().getConfiguration(); - ttl = config.getDefaultTTL(); - if (ttl != null && ttl == 0) { - ttl = (int) TimeUnit.DAYS.toSeconds(configuration.getDefaultStorageTtlDays()); - } - } - @Override public void onMsg(TbContext ctx, TbMsg msg) { withCallback(save(msg, ctx), aVoid -> ctx.tellSuccess(msg), e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); @@ -189,7 +175,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { query.append("?, "); } } - if (ttl != null && ttl > 0) { + if (config.getDefaultTtL() > 0) { query.append(" USING TTL ?"); } return query.toString(); @@ -233,8 +219,8 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } i.getAndIncrement(); }); - if (ttl != null && ttl > 0) { - stmtBuilder.setInt(i.get(), ttl); + if (config.getDefaultTtL() > 0) { + stmtBuilder.setInt(i.get(), config.getDefaultTtL()); } return getFuture(executeAsyncWrite(ctx, stmtBuilder.build()), rs -> null); } @@ -281,9 +267,9 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { boolean hasChanges = false; switch (fromVersion) { case 0: - if (!oldConfiguration.has("defaultTTL")) { + if (!oldConfiguration.has("defaultTtL")) { hasChanges = true; - ((ObjectNode) oldConfiguration).putNull("defaultTTL"); + ((ObjectNode) oldConfiguration).put("defaultTtL", 0); } break; default: diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java index 3efae275da..e391e178f5 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java @@ -27,13 +27,13 @@ public class TbSaveToCustomCassandraTableNodeConfiguration implements NodeConfig private String tableName; private Map fieldsMapping; - private Integer defaultTTL; + private int defaultTtL; @Override public TbSaveToCustomCassandraTableNodeConfiguration defaultConfiguration() { TbSaveToCustomCassandraTableNodeConfiguration configuration = new TbSaveToCustomCassandraTableNodeConfiguration(); - configuration.setDefaultTTL(0); + configuration.setDefaultTtL(0); configuration.setTableName(""); Map map = new HashMap<>(); map.put("", ""); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index c99da715d9..c90e8e2375 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -75,7 +75,7 @@ public class TbMsgTimeseriesNode implements TbNode { onTenantProfileUpdate(ctx.getTenantProfile()); } - private void onTenantProfileUpdate(TenantProfile tenantProfile) { + void onTenantProfileUpdate(TenantProfile tenantProfile) { DefaultTenantProfileConfiguration configuration = (DefaultTenantProfileConfiguration) tenantProfile.getProfileData().getConfiguration(); tenantProfileDefaultStorageTtl = TimeUnit.DAYS.toSeconds(configuration.getDefaultStorageTtlDays()); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index f3e7d0d004..2ef5d4173c 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -46,13 +46,9 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; -import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.msg.TbMsgType; -import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; -import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.cassandra.CassandraCluster; @@ -91,7 +87,6 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("ac4ca02e-2ae6-404a-8f7e-c4ae31c56aa7")); private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("64ad971e-9cfa-49e4-9f59-faa1a2350c6e")); - private final TenantProfileId TENANT_PROFILE_ID = new TenantProfileId(UUID.fromString("239af5ad-53c1-4179-802f-3821f92ed338")); private final ListeningExecutor dbCallbackExecutor = new TestDbCallbackExecutor(); @@ -140,7 +135,7 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad public void verifyDefaultConfig() { assertThat(config.getTableName()).isEqualTo(""); assertThat(config.getFieldsMapping()).isEqualTo(Map.of("", "")); - assertThat(config.getDefaultTTL()).isEqualTo(0); + assertThat(config.getDefaultTtL()).isEqualTo(0); } @Test @@ -175,7 +170,6 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); mockCassandraCluster(); - given(ctxMock.getTenantProfile()).willReturn(getTenantProfileWithTtl(5)); assertThatThrownBy(() -> node.init(ctxMock, configuration)) .isInstanceOf(TbNodeException.class) @@ -241,16 +235,14 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad @ParameterizedTest @MethodSource - public void givenTtl_whenOnMsg_thenVerifyStatement(Integer ttlFromConfig, - int ttlFromTenantProfileInDays, + public void givenTtl_whenOnMsg_thenVerifyStatement(int ttlFromConfig, String expectedQuery, Consumer verifyBuilder) throws TbNodeException { config.setTableName("readings"); config.setFieldsMapping(Map.of("$entityId", "entityIdTableColumn")); - config.setDefaultTTL(ttlFromConfig); + config.setDefaultTtL(ttlFromConfig); mockOnInit(); - given(ctxMock.getTenantProfile()).willReturn(getTenantProfileWithTtl(ttlFromTenantProfileInDays)); willAnswer(invocation -> boundStatementBuilderMock).given(node).getStmtBuilder(); given(boundStatementBuilderMock.setUuid(anyInt(), any(UUID.class))).willReturn(boundStatementBuilderMock); given(boundStatementBuilderMock.build()).willReturn(boundStatementMock); @@ -266,28 +258,20 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad private static Stream givenTtl_whenOnMsg_thenVerifyStatement() { return Stream.of( - Arguments.of(0, 0, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?)", + Arguments.of(0, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?)", (Consumer) builder -> { then(builder).should(never()).setInt(anyInt(), anyInt()); }), - Arguments.of(0, 5, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?) USING TTL ?", - (Consumer) builder -> { - then(builder).should().setInt(1, 432000); - }), - Arguments.of(20, 1, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?) USING TTL ?", + Arguments.of(20, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?) USING TTL ?", (Consumer) builder -> { then(builder).should().setInt(1, 20); - }), - Arguments.of(null, 2, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?)", - (Consumer) builder -> { - then(builder).should(never()).setInt(anyInt(), anyInt()); }) ); } @Test public void givenValidMsgStructure_whenOnMsg_thenVerifyMatchOfValuesInsertionOrderIntoStatementAndSaveToCustomCassandraTable() throws TbNodeException { - config.setDefaultTTL(25); + config.setDefaultTtL(25); config.setTableName("readings"); Map mappings = Map.of( "$entityId", "entityIdTableColumn", @@ -341,20 +325,19 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad Arguments.of(0, "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"}}", true, - "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTTL\":null}" + "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTtL\":0}" ), // default config for version 1 with upgrade from version 1 Arguments.of(1, - "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTTL\":0}", + "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTtL\":0}", false, - "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTTL\":0}" + "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTtL\":0}" ) ); } private void mockOnInit() { mockCassandraCluster(); - given(ctxMock.getTenantProfile()).willReturn(getTenantProfileWithTtl(5)); given(cassandraClusterMock.getDefaultWriteConsistencyLevel()).willReturn(DefaultConsistencyLevel.ONE); given(sessionMock.prepare(anyString())).willReturn(preparedStatementMock); } @@ -410,14 +393,4 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad then(boundStatementBuilderMock).should().setInt(values.size(), 25); } - private TenantProfile getTenantProfileWithTtl(int ttlInDays) { - var tenantProfile = new TenantProfile(TENANT_PROFILE_ID); - var tenantProfileData = new TenantProfileData(); - var tenantProfileConfiguration = new DefaultTenantProfileConfiguration(); - tenantProfileConfiguration.setDefaultStorageTtlDays(ttlInDays); - tenantProfileData.setConfiguration(tenantProfileConfiguration); - tenantProfile.setProfileData(tenantProfileData); - return tenantProfile; - } - } From baed618604d43c8a2de1d7cfa14c04d8cf0d7e39 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 27 Nov 2024 13:57:10 +0200 Subject: [PATCH 52/71] renamed property --- .../action/TbSaveToCustomCassandraTableNode.java | 10 +++++----- ...bSaveToCustomCassandraTableNodeConfiguration.java | 4 ++-- .../action/TbSaveToCustomCassandraTableNodeTest.java | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index 7d4cd64179..a0dbc5da97 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -175,7 +175,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { query.append("?, "); } } - if (config.getDefaultTtL() > 0) { + if (config.getDefaultTtl() > 0) { query.append(" USING TTL ?"); } return query.toString(); @@ -219,8 +219,8 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } i.getAndIncrement(); }); - if (config.getDefaultTtL() > 0) { - stmtBuilder.setInt(i.get(), config.getDefaultTtL()); + if (config.getDefaultTtl() > 0) { + stmtBuilder.setInt(i.get(), config.getDefaultTtl()); } return getFuture(executeAsyncWrite(ctx, stmtBuilder.build()), rs -> null); } @@ -267,9 +267,9 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { boolean hasChanges = false; switch (fromVersion) { case 0: - if (!oldConfiguration.has("defaultTtL")) { + if (!oldConfiguration.has("defaultTtl")) { hasChanges = true; - ((ObjectNode) oldConfiguration).put("defaultTtL", 0); + ((ObjectNode) oldConfiguration).put("defaultTtl", 0); } break; default: diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java index e391e178f5..fefe871f82 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java @@ -27,13 +27,13 @@ public class TbSaveToCustomCassandraTableNodeConfiguration implements NodeConfig private String tableName; private Map fieldsMapping; - private int defaultTtL; + private int defaultTtl; @Override public TbSaveToCustomCassandraTableNodeConfiguration defaultConfiguration() { TbSaveToCustomCassandraTableNodeConfiguration configuration = new TbSaveToCustomCassandraTableNodeConfiguration(); - configuration.setDefaultTtL(0); + configuration.setDefaultTtl(0); configuration.setTableName(""); Map map = new HashMap<>(); map.put("", ""); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index 2ef5d4173c..7df2a982de 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -135,7 +135,7 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad public void verifyDefaultConfig() { assertThat(config.getTableName()).isEqualTo(""); assertThat(config.getFieldsMapping()).isEqualTo(Map.of("", "")); - assertThat(config.getDefaultTtL()).isEqualTo(0); + assertThat(config.getDefaultTtl()).isEqualTo(0); } @Test @@ -240,7 +240,7 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad Consumer verifyBuilder) throws TbNodeException { config.setTableName("readings"); config.setFieldsMapping(Map.of("$entityId", "entityIdTableColumn")); - config.setDefaultTtL(ttlFromConfig); + config.setDefaultTtl(ttlFromConfig); mockOnInit(); willAnswer(invocation -> boundStatementBuilderMock).given(node).getStmtBuilder(); @@ -271,7 +271,7 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad @Test public void givenValidMsgStructure_whenOnMsg_thenVerifyMatchOfValuesInsertionOrderIntoStatementAndSaveToCustomCassandraTable() throws TbNodeException { - config.setDefaultTtL(25); + config.setDefaultTtl(25); config.setTableName("readings"); Map mappings = Map.of( "$entityId", "entityIdTableColumn", @@ -325,13 +325,13 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad Arguments.of(0, "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"}}", true, - "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTtL\":0}" + "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTtl\":0}" ), // default config for version 1 with upgrade from version 1 Arguments.of(1, - "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTtL\":0}", + "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTtl\":0}", false, - "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTtL\":0}" + "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTtl\":0}" ) ); } From 05690fce5d2972df74e5d9a9a1c830cd6b2639d1 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 28 Nov 2024 13:14:57 +0200 Subject: [PATCH 53/71] UI: Fixed mobile center bugs --- .../mobile-app-table-config.resolver.ts | 30 +++++++++++-------- .../applications/mobile-app.component.html | 4 +-- .../applications/mobile-app.component.ts | 5 ++-- .../remove-app-dialog.component.html | 2 +- .../remove-app-dialog.component.ts | 13 ++++++++ .../layout/custom-mobile-page.component.ts | 2 +- ...le-app-configuration-dialog.component.html | 12 +++++++- ...le-app-configuration-dialog.component.scss | 9 +++--- ...bile-app-configuration-dialog.component.ts | 26 +++++++++++++--- .../mobile-bundle-table-config.resolve.ts | 3 +- .../assets/locale/locale.constant-en_US.json | 8 ++++- 11 files changed, 85 insertions(+), 29 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/mobile/applications/mobile-app-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/mobile/applications/mobile-app-table-config.resolver.ts index 543ab89078..20ac575b2c 100644 --- a/ui-ngx/src/app/modules/home/pages/mobile/applications/mobile-app-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/mobile/applications/mobile-app-table-config.resolver.ts @@ -114,6 +114,8 @@ export class MobileAppTableConfigResolver { this.config.entitiesFetchFunction = pageLink => this.mobileAppService.getTenantMobileAppInfos(pageLink); this.config.loadEntity = id => this.mobileAppService.getMobileAppInfoById(id.id); this.config.saveEntity = (mobileApp) => this.mobileAppService.saveMobileApp(mobileApp); + this.config.deleteEntityTitle = (mobileApp) => this.translate.instant('mobile.delete-application-title-short', {name: mobileApp.name}); + this.config.deleteEntityContent = () => this.translate.instant('mobile.delete-application-text-short'); this.config.deleteEntity = id => this.mobileAppService.deleteMobileApp(id.id); this.config.cellActionDescriptors = this.configureCellActions(); @@ -138,19 +140,23 @@ export class MobileAppTableConfigResolver { if ($event) { $event.stopPropagation(); } - this.dialog.open(RemoveAppDialogComponent, { - disableClose: true, - panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], - data: { - id: entity.id.id - } - }).afterClosed() - .subscribe((res) => { - if (res) { - this.config.updateData(); + if(entity.status === MobileAppStatus.PUBLISHED) { + this.dialog.open(RemoveAppDialogComponent, { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data: { + id: entity.id.id } - }); + }).afterClosed() + .subscribe((res) => { + if (res) { + this.config.updateData(); + } + }); + } else { + this.config.getTable().deleteEntity($event, entity); + } } private mobileStatus(status: MobileAppStatus): string { diff --git a/ui-ngx/src/app/modules/home/pages/mobile/applications/mobile-app.component.html b/ui-ngx/src/app/modules/home/pages/mobile/applications/mobile-app.component.html index e74fb25b0b..36f52e8b71 100644 --- a/ui-ngx/src/app/modules/home/pages/mobile/applications/mobile-app.component.html +++ b/ui-ngx/src/app/modules/home/pages/mobile/applications/mobile-app.component.html @@ -148,7 +148,7 @@ mobile.sha256-certificate-fingerprints - + mobile.app-id - + { Validators.pattern(/^https?:\/\/play\.google\.com\/store\/apps\/details\?id=[a-zA-Z0-9._]+$/)], sha256CertFingerprints: [entity?.storeInfo?.sha256CertFingerprints ? entity.storeInfo.sha256CertFingerprints : '', Validators.pattern(/^[A-Fa-f0-9]{2}(:[A-Fa-f0-9]{2}){1,31}$/)], - appId: [entity?.storeInfo?.appId ? entity.storeInfo.appId : '', Validators.pattern(/^\d{7,10}$/)], + appId: [entity?.storeInfo?.appId ? entity.storeInfo.appId : '', Validators.pattern(/^[A-Z0-9]{10}\.[a-zA-Z0-9]+(\.[a-zA-Z0-9]+)*$/)], }), }); @@ -126,7 +126,8 @@ export class MobileAppComponent extends EntityComponent { override updateFormState(): void { super.updateFormState(); if (this.isEdit && this.entityForm && !this.isAdd) { - this.entityForm.get('status').updateValueAndValidity({onlySelf: false}); + this.entityForm.get('status').updateValueAndValidity({onlySelf: true}); + this.entityForm.get('platformType').updateValueAndValidity({onlySelf: true}); this.entityForm.get('platformType').disable({emitEvent: false}); if (this.entityForm.get('platformType').value === PlatformType.ANDROID) { this.entityForm.get('storeInfo.appId').disable({emitEvent: false}); diff --git a/ui-ngx/src/app/modules/home/pages/mobile/applications/remove-app-dialog.component.html b/ui-ngx/src/app/modules/home/pages/mobile/applications/remove-app-dialog.component.html index faca470cdf..4f1569468f 100644 --- a/ui-ngx/src/app/modules/home/pages/mobile/applications/remove-app-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/mobile/applications/remove-app-dialog.component.html @@ -36,7 +36,7 @@ +