From ada9f8b7ebd422a6918b5abcc7d11659bcdce046 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 16 Apr 2024 15:40:34 +0300 Subject: [PATCH 01/37] added tests for save to custom table node --- .../TbSaveToCustomCassandraTableNode.java | 8 +- .../TbSaveToCustomCassandraTableNodeTest.java | 248 ++++++++++++++++++ 2 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index 4b957db9cd..1e55e9d320 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -173,7 +173,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { throw new IllegalStateException("Invalid message structure, it is not a JSON Object:" + data); } else { JsonObject dataAsObject = data.getAsJsonObject(); - BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(saveStmt.bind()); + BoundStatementBuilder stmtBuilder = getStmtBuilder(); AtomicInteger i = new AtomicInteger(0); fieldsMap.forEach((key, value) -> { if (key.equals(ENTITY_ID)) { @@ -198,7 +198,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } else if (dataKeyElement.isJsonObject()) { stmtBuilder.setString(i.get(), dataKeyElement.getAsJsonObject().toString()); } else { - throw new IllegalStateException("Message data key: '" + key + "' with value: '" + value + "' is not a JSON Object or JSON Primitive!"); + throw new IllegalStateException("Message data key: '" + key + "' with value: '" + dataKeyElement + "' is not a JSON Object or JSON Primitive!"); } } else { throw new RuntimeException("Message data doesn't contain key: " + "'" + key + "'!"); @@ -209,6 +209,10 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } } + BoundStatementBuilder getStmtBuilder() { + return new BoundStatementBuilder(saveStmt.bind()); + } + private TbResultSetFuture executeAsyncWrite(TbContext ctx, Statement statement) { return executeAsync(ctx, statement, defaultWriteLevel); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java new file mode 100644 index 0000000000..699af15fa3 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -0,0 +1,248 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.action; + +import com.datastax.oss.driver.api.core.DefaultConsistencyLevel; +import com.datastax.oss.driver.api.core.ProtocolVersion; +import com.datastax.oss.driver.api.core.cql.BoundStatement; +import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; +import com.datastax.oss.driver.api.core.cql.ColumnDefinitions; +import com.datastax.oss.driver.api.core.cql.PreparedStatement; +import com.datastax.oss.driver.api.core.metadata.Node; +import com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry; + +import com.google.common.util.concurrent.SettableFuture; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ListeningExecutor; +import org.thingsboard.rule.engine.TestDbCallbackExecutor; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cassandra.guava.GuavaSession; +import org.thingsboard.server.dao.nosql.CassandraStatementTask; +import org.thingsboard.server.dao.nosql.TbResultSet; +import org.thingsboard.server.dao.nosql.TbResultSetFuture; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static java.util.Collections.emptyMap; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class TbSaveToCustomCassandraTableNodeTest { + + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("ac4ca02e-2ae6-404a-8f7e-c4ae31c56aa7")); + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("64ad971e-9cfa-49e4-9f59-faa1a2350c6e")); + + private final ListeningExecutor dbCallbackExecutor = new TestDbCallbackExecutor(); + + private TbSaveToCustomCassandraTableNode node; + private TbSaveToCustomCassandraTableNodeConfiguration config; + + @Mock + private TbContext ctxMock; + @Mock + private CassandraCluster cassandraClusterMock; + @Mock + private GuavaSession sessionMock; + @Mock + private PreparedStatement preparedStatementMock; + @Mock + private BoundStatement boundStatementMock; + @Mock + private BoundStatementBuilder boundStatementBuilderMock; + @Mock + private ColumnDefinitions columnDefinitionsMock; + @Mock + private CodecRegistry codecRegistryMock; + @Mock + private ProtocolVersion protocolVersionMock; + @Mock + private Node nodeMock; + + @Test + void givenCassandraClusterIsMissing_whenInit_thenThrowsException() { + node = new TbSaveToCustomCassandraTableNode(); + config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); + var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); + assertThatThrownBy(() -> node.init(ctxMock, configuration)) + .isInstanceOf(RuntimeException.class) + .hasMessage("Unable to connect to Cassandra database"); + } + + @Test + void givenFieldsMapIsEmpty_whenInit_thenThrowsException() { + node = new TbSaveToCustomCassandraTableNode(); + config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); + config.setFieldsMapping(emptyMap()); + var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); + + when(ctxMock.getCassandraCluster()).thenReturn(cassandraClusterMock); + + assertThatThrownBy(() -> node.init(ctxMock, configuration)) + .isInstanceOf(RuntimeException.class) + .hasMessage("Fields(key,value) map is empty!"); + } + + @Test + void givenInvalidMessageStructure_whenOnMsg_thenThrowsException() throws TbNodeException { + node = new TbSaveToCustomCassandraTableNode(); + config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); + config.setTableName("temperature_sensor"); + config.setFieldsMapping(Map.of("temp", "temperature")); + var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); + + initializeMocks(); + + node.init(ctxMock, configuration); + + String data = ""; + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data); + + assertThatThrownBy(() -> node.onMsg(ctxMock, msg)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Invalid message structure, it is not a JSON Object:" + null); + } + + @Test + void givenDataKeyIsMissingInMsg_whenOnMsg_thenThrowsException() throws TbNodeException { + node = new TbSaveToCustomCassandraTableNode(); + config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); + config.setTableName("temperature_sensor"); + config.setFieldsMapping(Map.of("temp", "temperature")); + var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); + + initializeMocks(); + when(preparedStatementMock.bind()).thenReturn(boundStatementMock); + when(boundStatementMock.getPreparedStatement()).thenReturn(preparedStatementMock); + when(preparedStatementMock.getVariableDefinitions()).thenReturn(columnDefinitionsMock); + when(boundStatementMock.codecRegistry()).thenReturn(codecRegistryMock); + when(boundStatementMock.protocolVersion()).thenReturn(protocolVersionMock); + when(boundStatementMock.getNode()).thenReturn(nodeMock); + + node.init(ctxMock, configuration); + + String data = """ + { + "humidity": 77 + } + """; + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data);; + + assertThatThrownBy(() -> node.onMsg(ctxMock, msg)) + .isInstanceOf(RuntimeException.class) + .hasMessage("Message data doesn't contain key: 'temp'!"); + } + + @Test + void givenUnsupportedData_whenOnMsg_thenThrowsException() throws TbNodeException { + node = new TbSaveToCustomCassandraTableNode(); + config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); + config.setTableName("temperature_sensor"); + config.setFieldsMapping(Map.of("temp", "temperature")); + var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); + + initializeMocks(); + when(preparedStatementMock.bind()).thenReturn(boundStatementMock); + when(boundStatementMock.getPreparedStatement()).thenReturn(preparedStatementMock); + when(preparedStatementMock.getVariableDefinitions()).thenReturn(columnDefinitionsMock); + when(boundStatementMock.getValues()).thenReturn(List.of(ByteBuffer.allocate(1))); + when(boundStatementMock.codecRegistry()).thenReturn(codecRegistryMock); + when(boundStatementMock.protocolVersion()).thenReturn(protocolVersionMock); + when(boundStatementMock.getNode()).thenReturn(nodeMock); + + node.init(ctxMock, configuration); + + String data = """ + { + "temp": [value] + } + """; + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data); + + + assertThatThrownBy(() -> node.onMsg(ctxMock, msg)) + .isInstanceOf(RuntimeException.class) + .hasMessage("Message data key: 'temp' with value: '[\"value\"]' is not a JSON Object or JSON Primitive!"); + } + + @Test + void givenValidMsgStructure_whenOnMsg_thenOk() throws Exception { + node = spy(new TbSaveToCustomCassandraTableNode()); + config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); + config.setTableName("temperature_sensor"); + config.setFieldsMapping(Map.of("temp", "temperature")); + var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); + + initializeMocks(); + doAnswer(invocation -> boundStatementBuilderMock).when(node).getStmtBuilder(); + when(boundStatementBuilderMock.setLong(anyInt(), anyLong())).thenReturn(boundStatementBuilderMock); + when(boundStatementBuilderMock.build()).thenReturn(boundStatementMock); + when(ctxMock.getTenantId()).thenReturn(TENANT_ID); + doAnswer(invocation -> { + SettableFuture mainFuture = SettableFuture.create(); + mainFuture.set(new TbResultSet(null, null, null)); + TbResultSetFuture value = new TbResultSetFuture(mainFuture); + return value; + }).when(ctxMock).submitCassandraWriteTask(any()); + when(ctxMock.getDbCallbackExecutor()).thenReturn(dbCallbackExecutor); + + node.init(ctxMock, configuration); + + String data = """ + { + "temp": 31, + "humidity": 77 + } + """; + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data); + + node.onMsg(ctxMock, msg); + + verify(sessionMock).prepare(eq("INSERT INTO cs_tb_temperature_sensor(temperature) VALUES(?)")); + verify(ctxMock).submitCassandraWriteTask(any(CassandraStatementTask.class)); + verify(ctxMock).tellSuccess(eq(msg)); + } + + private void initializeMocks() { + when(ctxMock.getCassandraCluster()).thenReturn(cassandraClusterMock); + when(cassandraClusterMock.getSession()).thenReturn(sessionMock); + when(cassandraClusterMock.getDefaultWriteConsistencyLevel()).thenReturn(DefaultConsistencyLevel.ONE); + when(sessionMock.prepare(anyString())).thenReturn(preparedStatementMock); + } +} From 00251f11b1dd871981f01b101d7cec34920d7cfe Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 29 May 2024 11:19:15 +0300 Subject: [PATCH 02/37] refactored tests --- .../TbSaveToCustomCassandraTableNodeTest.java | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index 699af15fa3..d760568bd4 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -25,6 +25,7 @@ import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry; import com.google.common.util.concurrent.SettableFuture; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -95,10 +96,14 @@ public class TbSaveToCustomCassandraTableNodeTest { @Mock private Node nodeMock; + @BeforeEach + void setUp() { + node = spy(new TbSaveToCustomCassandraTableNode()); + config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); + } + @Test void givenCassandraClusterIsMissing_whenInit_thenThrowsException() { - node = new TbSaveToCustomCassandraTableNode(); - config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); assertThatThrownBy(() -> node.init(ctxMock, configuration)) .isInstanceOf(RuntimeException.class) @@ -107,8 +112,6 @@ public class TbSaveToCustomCassandraTableNodeTest { @Test void givenFieldsMapIsEmpty_whenInit_thenThrowsException() { - node = new TbSaveToCustomCassandraTableNode(); - config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); config.setFieldsMapping(emptyMap()); var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); @@ -121,8 +124,6 @@ public class TbSaveToCustomCassandraTableNodeTest { @Test void givenInvalidMessageStructure_whenOnMsg_thenThrowsException() throws TbNodeException { - node = new TbSaveToCustomCassandraTableNode(); - config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); config.setTableName("temperature_sensor"); config.setFieldsMapping(Map.of("temp", "temperature")); var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); @@ -131,8 +132,7 @@ public class TbSaveToCustomCassandraTableNodeTest { node.init(ctxMock, configuration); - String data = ""; - TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data); + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); assertThatThrownBy(() -> node.onMsg(ctxMock, msg)) .isInstanceOf(IllegalStateException.class) @@ -141,8 +141,6 @@ public class TbSaveToCustomCassandraTableNodeTest { @Test void givenDataKeyIsMissingInMsg_whenOnMsg_thenThrowsException() throws TbNodeException { - node = new TbSaveToCustomCassandraTableNode(); - config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); config.setTableName("temperature_sensor"); config.setFieldsMapping(Map.of("temp", "temperature")); var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); @@ -171,8 +169,6 @@ public class TbSaveToCustomCassandraTableNodeTest { @Test void givenUnsupportedData_whenOnMsg_thenThrowsException() throws TbNodeException { - node = new TbSaveToCustomCassandraTableNode(); - config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); config.setTableName("temperature_sensor"); config.setFieldsMapping(Map.of("temp", "temperature")); var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); @@ -203,8 +199,6 @@ public class TbSaveToCustomCassandraTableNodeTest { @Test void givenValidMsgStructure_whenOnMsg_thenOk() throws Exception { - node = spy(new TbSaveToCustomCassandraTableNode()); - config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); config.setTableName("temperature_sensor"); config.setFieldsMapping(Map.of("temp", "temperature")); var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); From 6a9a933d4869475556d67d3fd3cad1526692bbeb Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 6 Jun 2024 16:18:43 +0300 Subject: [PATCH 03/37] added tests for check builder --- .../TbSaveToCustomCassandraTableNodeTest.java | 187 ++++++++++++------ 1 file changed, 131 insertions(+), 56 deletions(-) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index d760568bd4..9d0c646828 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -25,9 +25,14 @@ import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry; import com.google.common.util.concurrent.SettableFuture; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.JacksonUtil; @@ -47,14 +52,19 @@ import org.thingsboard.server.dao.nosql.CassandraStatementTask; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.nosql.TbResultSetFuture; -import java.nio.ByteBuffer; -import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.stream.Stream; import static java.util.Collections.emptyMap; -import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyDouble; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; @@ -102,6 +112,11 @@ public class TbSaveToCustomCassandraTableNodeTest { config = new TbSaveToCustomCassandraTableNodeConfiguration().defaultConfiguration(); } + @AfterEach + void tearDown() { + node.destroy(); + } + @Test void givenCassandraClusterIsMissing_whenInit_thenThrowsException() { var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); @@ -126,14 +141,12 @@ public class TbSaveToCustomCassandraTableNodeTest { void givenInvalidMessageStructure_whenOnMsg_thenThrowsException() throws TbNodeException { config.setTableName("temperature_sensor"); config.setFieldsMapping(Map.of("temp", "temperature")); - var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); - initializeMocks(); + mockOnInit(); - node.init(ctxMock, configuration); + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); - assertThatThrownBy(() -> node.onMsg(ctxMock, msg)) .isInstanceOf(IllegalStateException.class) .hasMessage("Invalid message structure, it is not a JSON Object:" + null); @@ -143,25 +156,18 @@ public class TbSaveToCustomCassandraTableNodeTest { void givenDataKeyIsMissingInMsg_whenOnMsg_thenThrowsException() throws TbNodeException { config.setTableName("temperature_sensor"); config.setFieldsMapping(Map.of("temp", "temperature")); - var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); - initializeMocks(); - when(preparedStatementMock.bind()).thenReturn(boundStatementMock); - when(boundStatementMock.getPreparedStatement()).thenReturn(preparedStatementMock); - when(preparedStatementMock.getVariableDefinitions()).thenReturn(columnDefinitionsMock); - when(boundStatementMock.codecRegistry()).thenReturn(codecRegistryMock); - when(boundStatementMock.protocolVersion()).thenReturn(protocolVersionMock); - when(boundStatementMock.getNode()).thenReturn(nodeMock); + mockOnInit(); + mockBoundStatement(); - node.init(ctxMock, configuration); + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); String data = """ { "humidity": 77 } """; - TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data);; - + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data); assertThatThrownBy(() -> node.onMsg(ctxMock, msg)) .isInstanceOf(RuntimeException.class) .hasMessage("Message data doesn't contain key: 'temp'!"); @@ -171,18 +177,11 @@ public class TbSaveToCustomCassandraTableNodeTest { void givenUnsupportedData_whenOnMsg_thenThrowsException() throws TbNodeException { config.setTableName("temperature_sensor"); config.setFieldsMapping(Map.of("temp", "temperature")); - var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); - initializeMocks(); - when(preparedStatementMock.bind()).thenReturn(boundStatementMock); - when(boundStatementMock.getPreparedStatement()).thenReturn(preparedStatementMock); - when(preparedStatementMock.getVariableDefinitions()).thenReturn(columnDefinitionsMock); - when(boundStatementMock.getValues()).thenReturn(List.of(ByteBuffer.allocate(1))); - when(boundStatementMock.codecRegistry()).thenReturn(codecRegistryMock); - when(boundStatementMock.protocolVersion()).thenReturn(protocolVersionMock); - when(boundStatementMock.getNode()).thenReturn(nodeMock); + mockOnInit(); + mockBoundStatement(); - node.init(ctxMock, configuration); + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); String data = """ { @@ -190,53 +189,129 @@ public class TbSaveToCustomCassandraTableNodeTest { } """; TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data); - - assertThatThrownBy(() -> node.onMsg(ctxMock, msg)) .isInstanceOf(RuntimeException.class) .hasMessage("Message data key: 'temp' with value: '[\"value\"]' is not a JSON Object or JSON Primitive!"); } @Test - void givenValidMsgStructure_whenOnMsg_thenOk() throws Exception { - config.setTableName("temperature_sensor"); - config.setFieldsMapping(Map.of("temp", "temperature")); - var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); + void givenValidMsgStructureAndKeyIsEntityId_whenOnMsg_thenOk() throws Exception { + config.setTableName("readings"); + config.setFieldsMapping(Map.of("$entityId", "entityId")); - initializeMocks(); - doAnswer(invocation -> boundStatementBuilderMock).when(node).getStmtBuilder(); - when(boundStatementBuilderMock.setLong(anyInt(), anyLong())).thenReturn(boundStatementBuilderMock); - when(boundStatementBuilderMock.build()).thenReturn(boundStatementMock); - when(ctxMock.getTenantId()).thenReturn(TENANT_ID); - doAnswer(invocation -> { - SettableFuture mainFuture = SettableFuture.create(); - mainFuture.set(new TbResultSet(null, null, null)); - TbResultSetFuture value = new TbResultSetFuture(mainFuture); - return value; - }).when(ctxMock).submitCassandraWriteTask(any()); - when(ctxMock.getDbCallbackExecutor()).thenReturn(dbCallbackExecutor); + mockOnInit(); + when(boundStatementBuilderMock.setUuid(anyInt(), any(UUID.class))).thenReturn(boundStatementBuilderMock); + when(boundStatementMock.getConsistencyLevel()).thenReturn(DefaultConsistencyLevel.ONE); + mockBoundStatementBuilder(); + mockSubmittingCassandraTask(); - node.init(ctxMock, configuration); + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); - String data = """ + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, TbMsg.EMPTY_JSON_OBJECT); + node.onMsg(ctxMock, msg); + + verify(sessionMock).prepare("INSERT INTO cs_tb_readings(entityId) VALUES(?)"); + verify(boundStatementBuilderMock).setUuid(anyInt(), eq(DEVICE_ID.getId())); + + ArgumentCaptor taskCaptor = ArgumentCaptor.forClass(CassandraStatementTask.class); + verify(ctxMock).submitCassandraWriteTask(taskCaptor.capture()); + CassandraStatementTask task = taskCaptor.getValue(); + assertThat(task.getTenantId()).isEqualTo(TENANT_ID); + assertThat(task.getSession()).isEqualTo(sessionMock); + assertThat(task.getStatement()).isEqualTo(boundStatementMock); + await().atMost(1, TimeUnit.SECONDS).untilAsserted( + () -> verify(ctxMock).tellSuccess(msg) + ); + } + + @ParameterizedTest + @MethodSource + void givenValidMsgStructure_whenOnMsg_thenSaveToCustomCassandraTable( + String valueFromData, + Consumer mockBuilderConsumer, + Consumer verifyBuilderConsumer) throws TbNodeException { + config.setTableName("sensor_readings"); + config.setFieldsMapping(Map.of("msgField1", "tableColumn1", "msgField2", "tableColumn2")); + + mockOnInit(); + mockBuilderConsumer.accept(boundStatementBuilderMock); + mockBoundStatementBuilder(); + mockSubmittingCassandraTask(); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + String data = String.format(""" { - "temp": 31, - "humidity": 77 + "msgField1": %s, + "msgField2": "value2", + "anotherField": 22 } - """; + """, valueFromData); TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data); - node.onMsg(ctxMock, msg); - verify(sessionMock).prepare(eq("INSERT INTO cs_tb_temperature_sensor(temperature) VALUES(?)")); - verify(ctxMock).submitCassandraWriteTask(any(CassandraStatementTask.class)); - verify(ctxMock).tellSuccess(eq(msg)); + verifyBuilderConsumer.accept(boundStatementBuilderMock); + + ArgumentCaptor taskCaptor = ArgumentCaptor.forClass(CassandraStatementTask.class); + verify(ctxMock).submitCassandraWriteTask(taskCaptor.capture()); + CassandraStatementTask task = taskCaptor.getValue(); + assertThat(task.getTenantId()).isEqualTo(TENANT_ID); + assertThat(task.getSession()).isEqualTo(sessionMock); + assertThat(task.getStatement()).isEqualTo(boundStatementMock); + await().atMost(1, TimeUnit.SECONDS).untilAsserted( + () -> verify(ctxMock).tellSuccess(msg) + ); + } + + private static Stream givenValidMsgStructure_whenOnMsg_thenSaveToCustomCassandraTable() { + return Stream.of( + Arguments.of("1.0", + (Consumer) mockBuilder -> when(mockBuilder.setDouble(anyInt(), anyDouble())).thenReturn(mockBuilder), + (Consumer) verifyBuilder -> verify(verifyBuilder).setDouble(anyInt(), eq(1.0))), + Arguments.of("2", + (Consumer) mockBuilder -> when(mockBuilder.setLong(anyInt(), anyLong())).thenReturn(mockBuilder), + (Consumer) verifyBuilder -> verify(verifyBuilder).setLong(anyInt(), eq(2L))), + Arguments.of("true", + (Consumer) mockBuilder -> when(mockBuilder.setBoolean(anyInt(), anyBoolean())).thenReturn(mockBuilder), + (Consumer) verifyBuilder -> verify(verifyBuilder).setBoolean(anyInt(), eq(true))), + Arguments.of("\"string value\"", + (Consumer) mockBuilder -> when(mockBuilder.setString(anyInt(), anyString())).thenReturn(mockBuilder), + (Consumer) verifyBuilder -> verify(verifyBuilder).setString(anyInt(), eq("string value"))), + Arguments.of("{\"key\":\"value\"}", + (Consumer) mockBuilder -> when(mockBuilder.setString(anyInt(), anyString())).thenReturn(mockBuilder), + (Consumer) verifyBuilder -> verify(verifyBuilder).setString(anyInt(), eq("{\"key\":\"value\"}"))) + ); } - private void initializeMocks() { + private void mockOnInit() { when(ctxMock.getCassandraCluster()).thenReturn(cassandraClusterMock); when(cassandraClusterMock.getSession()).thenReturn(sessionMock); when(cassandraClusterMock.getDefaultWriteConsistencyLevel()).thenReturn(DefaultConsistencyLevel.ONE); when(sessionMock.prepare(anyString())).thenReturn(preparedStatementMock); } + + private void mockBoundStatement() { + when(preparedStatementMock.bind()).thenReturn(boundStatementMock); + when(boundStatementMock.getPreparedStatement()).thenReturn(preparedStatementMock); + when(preparedStatementMock.getVariableDefinitions()).thenReturn(columnDefinitionsMock); + when(boundStatementMock.codecRegistry()).thenReturn(codecRegistryMock); + when(boundStatementMock.protocolVersion()).thenReturn(protocolVersionMock); + when(boundStatementMock.getNode()).thenReturn(nodeMock); + } + + private void mockBoundStatementBuilder() { + doAnswer(invocation -> boundStatementBuilderMock).when(node).getStmtBuilder(); + when(boundStatementBuilderMock.build()).thenReturn(boundStatementMock); + } + + private void mockSubmittingCassandraTask() { + when(ctxMock.getTenantId()).thenReturn(TENANT_ID); + doAnswer(invocation -> { + SettableFuture mainFuture = SettableFuture.create(); + mainFuture.set(new TbResultSet(null, null, null)); + return new TbResultSetFuture(mainFuture); + }).when(ctxMock).submitCassandraWriteTask(any()); + when(ctxMock.getDbCallbackExecutor()).thenReturn(dbCallbackExecutor); + } + } From 3368b9480b8ea6f49e8657209039cae8ae00f586 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 6 Jun 2024 16:29:38 +0300 Subject: [PATCH 04/37] added test for default config --- .../engine/action/TbSaveToCustomCassandraTableNodeTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index 9d0c646828..dab748b0a4 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -117,6 +117,12 @@ public class TbSaveToCustomCassandraTableNodeTest { node.destroy(); } + @Test + void verifyDefaultConfig() { + assertThat(config.getTableName()).isEqualTo(""); + assertThat(config.getFieldsMapping()).isEqualTo(Map.of("", "")); + } + @Test void givenCassandraClusterIsMissing_whenInit_thenThrowsException() { var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); From c137a144816c47446080925750ca5edcef80d020 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 7 Jun 2024 18:03:45 +0300 Subject: [PATCH 05/37] created complex test instead of parametrized --- .../TbSaveToCustomCassandraTableNode.java | 4 +- .../TbSaveToCustomCassandraTableNodeTest.java | 114 ++++++------------ 2 files changed, 38 insertions(+), 80 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index 1e55e9d320..d262edbb44 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -88,7 +88,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { config = TbNodeUtils.convert(configuration, TbSaveToCustomCassandraTableNodeConfiguration.class); cassandraCluster = ctx.getCassandraCluster(); if (cassandraCluster == null) { - throw new RuntimeException("Unable to connect to Cassandra database"); + throw new TbNodeException("Unable to connect to Cassandra database"); } else { startExecutor(); saveStmt = getSaveStmt(); @@ -170,7 +170,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { private ListenableFuture save(TbMsg msg, TbContext ctx) { JsonElement data = parser.parse(msg.getData()); if (!data.isJsonObject()) { - throw new IllegalStateException("Invalid message structure, it is not a JSON Object:" + data); + throw new IllegalStateException("Invalid message structure, it is not a JSON Object: " + data); } else { JsonObject dataAsObject = data.getAsJsonObject(); BoundStatementBuilder stmtBuilder = getStmtBuilder(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index dab748b0a4..d841928041 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -23,15 +23,11 @@ import com.datastax.oss.driver.api.core.cql.ColumnDefinitions; import com.datastax.oss.driver.api.core.cql.PreparedStatement; import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry; - import com.google.common.util.concurrent.SettableFuture; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -54,14 +50,10 @@ import org.thingsboard.server.dao.nosql.TbResultSetFuture; import java.util.Map; import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; -import java.util.stream.Stream; import static java.util.Collections.emptyMap; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyDouble; @@ -127,7 +119,7 @@ public class TbSaveToCustomCassandraTableNodeTest { void givenCassandraClusterIsMissing_whenInit_thenThrowsException() { var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); assertThatThrownBy(() -> node.init(ctxMock, configuration)) - .isInstanceOf(RuntimeException.class) + .isInstanceOf(TbNodeException.class) .hasMessage("Unable to connect to Cassandra database"); } @@ -155,7 +147,7 @@ public class TbSaveToCustomCassandraTableNodeTest { TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); assertThatThrownBy(() -> node.onMsg(ctxMock, msg)) .isInstanceOf(IllegalStateException.class) - .hasMessage("Invalid message structure, it is not a JSON Object:" + null); + .hasMessage("Invalid message structure, it is not a JSON Object: " + null); } @Test @@ -201,92 +193,44 @@ public class TbSaveToCustomCassandraTableNodeTest { } @Test - void givenValidMsgStructureAndKeyIsEntityId_whenOnMsg_thenOk() throws Exception { + void givenValidMsgStructure_whenOnMsg_thenSaveToCustomCassandraTable() throws TbNodeException { config.setTableName("readings"); - config.setFieldsMapping(Map.of("$entityId", "entityId")); - - mockOnInit(); - when(boundStatementBuilderMock.setUuid(anyInt(), any(UUID.class))).thenReturn(boundStatementBuilderMock); - when(boundStatementMock.getConsistencyLevel()).thenReturn(DefaultConsistencyLevel.ONE); - mockBoundStatementBuilder(); - mockSubmittingCassandraTask(); - - node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); - - TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, TbMsg.EMPTY_JSON_OBJECT); - node.onMsg(ctxMock, msg); - - verify(sessionMock).prepare("INSERT INTO cs_tb_readings(entityId) VALUES(?)"); - verify(boundStatementBuilderMock).setUuid(anyInt(), eq(DEVICE_ID.getId())); - - ArgumentCaptor taskCaptor = ArgumentCaptor.forClass(CassandraStatementTask.class); - verify(ctxMock).submitCassandraWriteTask(taskCaptor.capture()); - CassandraStatementTask task = taskCaptor.getValue(); - assertThat(task.getTenantId()).isEqualTo(TENANT_ID); - assertThat(task.getSession()).isEqualTo(sessionMock); - assertThat(task.getStatement()).isEqualTo(boundStatementMock); - await().atMost(1, TimeUnit.SECONDS).untilAsserted( - () -> verify(ctxMock).tellSuccess(msg) - ); - } - - @ParameterizedTest - @MethodSource - void givenValidMsgStructure_whenOnMsg_thenSaveToCustomCassandraTable( - String valueFromData, - Consumer mockBuilderConsumer, - Consumer verifyBuilderConsumer) throws TbNodeException { - config.setTableName("sensor_readings"); - config.setFieldsMapping(Map.of("msgField1", "tableColumn1", "msgField2", "tableColumn2")); + config.setFieldsMapping(Map.of( + "$entityId", "entityIdTableColumn", + "doubleField", "doubleTableColumn", + "longField", "longTableColumn", + "booleanField", "booleanTableColumn", + "stringField", "stringTableColumn", + "jsonField", "jsonTableColumn" + )); mockOnInit(); - mockBuilderConsumer.accept(boundStatementBuilderMock); mockBoundStatementBuilder(); mockSubmittingCassandraTask(); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); - - String data = String.format(""" + String data = """ { - "msgField1": %s, - "msgField2": "value2", - "anotherField": 22 + "doubleField": 22.5, + "longField": 56, + "booleanField": true, + "stringField": "some string", + "jsonField": { + "key": "value" + } } - """, valueFromData); + """; TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, data); node.onMsg(ctxMock, msg); - verifyBuilderConsumer.accept(boundStatementBuilderMock); - + verifySettingStatementBuilder(); ArgumentCaptor taskCaptor = ArgumentCaptor.forClass(CassandraStatementTask.class); verify(ctxMock).submitCassandraWriteTask(taskCaptor.capture()); CassandraStatementTask task = taskCaptor.getValue(); assertThat(task.getTenantId()).isEqualTo(TENANT_ID); assertThat(task.getSession()).isEqualTo(sessionMock); assertThat(task.getStatement()).isEqualTo(boundStatementMock); - await().atMost(1, TimeUnit.SECONDS).untilAsserted( - () -> verify(ctxMock).tellSuccess(msg) - ); - } - - private static Stream givenValidMsgStructure_whenOnMsg_thenSaveToCustomCassandraTable() { - return Stream.of( - Arguments.of("1.0", - (Consumer) mockBuilder -> when(mockBuilder.setDouble(anyInt(), anyDouble())).thenReturn(mockBuilder), - (Consumer) verifyBuilder -> verify(verifyBuilder).setDouble(anyInt(), eq(1.0))), - Arguments.of("2", - (Consumer) mockBuilder -> when(mockBuilder.setLong(anyInt(), anyLong())).thenReturn(mockBuilder), - (Consumer) verifyBuilder -> verify(verifyBuilder).setLong(anyInt(), eq(2L))), - Arguments.of("true", - (Consumer) mockBuilder -> when(mockBuilder.setBoolean(anyInt(), anyBoolean())).thenReturn(mockBuilder), - (Consumer) verifyBuilder -> verify(verifyBuilder).setBoolean(anyInt(), eq(true))), - Arguments.of("\"string value\"", - (Consumer) mockBuilder -> when(mockBuilder.setString(anyInt(), anyString())).thenReturn(mockBuilder), - (Consumer) verifyBuilder -> verify(verifyBuilder).setString(anyInt(), eq("string value"))), - Arguments.of("{\"key\":\"value\"}", - (Consumer) mockBuilder -> when(mockBuilder.setString(anyInt(), anyString())).thenReturn(mockBuilder), - (Consumer) verifyBuilder -> verify(verifyBuilder).setString(anyInt(), eq("{\"key\":\"value\"}"))) - ); + verify(ctxMock).tellSuccess(msg); } private void mockOnInit() { @@ -307,6 +251,11 @@ public class TbSaveToCustomCassandraTableNodeTest { private void mockBoundStatementBuilder() { doAnswer(invocation -> boundStatementBuilderMock).when(node).getStmtBuilder(); + when(boundStatementBuilderMock.setUuid(anyInt(), any(UUID.class))).thenReturn(boundStatementBuilderMock); + when(boundStatementBuilderMock.setDouble(anyInt(), anyDouble())).thenReturn(boundStatementBuilderMock); + when(boundStatementBuilderMock.setLong(anyInt(), anyLong())).thenReturn(boundStatementBuilderMock); + when(boundStatementBuilderMock.setBoolean(anyInt(), anyBoolean())).thenReturn(boundStatementBuilderMock); + when(boundStatementBuilderMock.setString(anyInt(), anyString())).thenReturn(boundStatementBuilderMock); when(boundStatementBuilderMock.build()).thenReturn(boundStatementMock); } @@ -320,4 +269,13 @@ public class TbSaveToCustomCassandraTableNodeTest { when(ctxMock.getDbCallbackExecutor()).thenReturn(dbCallbackExecutor); } + private void verifySettingStatementBuilder() { + verify(boundStatementBuilderMock).setUuid(anyInt(), eq(DEVICE_ID.getId())); + verify(boundStatementBuilderMock).setDouble(anyInt(), eq(22.5)); + verify(boundStatementBuilderMock).setLong(anyInt(), eq(56L)); + verify(boundStatementBuilderMock).setBoolean(anyInt(), eq(true)); + verify(boundStatementBuilderMock).setString(anyInt(), eq("some string")); + verify(boundStatementBuilderMock).setString(anyInt(), eq("{\"key\":\"value\"}")); + } + } From e561e268719dc353b15d03c0c0caf9fccf2abeb2 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 11 Jun 2024 10:41:12 +0300 Subject: [PATCH 06/37] added possibility to set ttl via config --- .../TbSaveToCustomCassandraTableNode.java | 52 ++++++++++++++++--- ...CustomCassandraTableNodeConfiguration.java | 2 + 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index 1aad815dfa..9a3950a1c2 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -21,6 +21,8 @@ import com.datastax.oss.driver.api.core.cql.BoundStatement; import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.datastax.oss.driver.api.core.cql.PreparedStatement; import com.datastax.oss.driver.api.core.cql.Statement; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Function; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -28,6 +30,7 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.gson.JsonPrimitive; +import jakarta.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.TbContext; @@ -35,20 +38,23 @@ import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.dao.cassandra.CassandraCluster; import org.thingsboard.server.dao.cassandra.guava.GuavaSession; import org.thingsboard.server.dao.nosql.CassandraStatementTask; import org.thingsboard.server.dao.nosql.TbResultSetFuture; -import jakarta.annotation.Nullable; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import static org.thingsboard.common.util.DonAsynchron.withCallback; @@ -57,6 +63,7 @@ import static org.thingsboard.common.util.DonAsynchron.withCallback; @RuleNode(type = ComponentType.ACTION, name = "save to custom table", configClazz = TbSaveToCustomCassandraTableNodeConfiguration.class, + version = 1, nodeDescription = "Node stores data from incoming Message payload to the Cassandra database into the predefined custom table" + " that should have cs_tb_ prefix, to avoid the data insertion to the common TB tables.
" + "Note: rule node can be used only for Cassandra DB.", @@ -81,6 +88,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { private PreparedStatement saveStmt; private ExecutorService readResultsProcessingExecutor; private Map fieldsMap; + private long ttl; @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { @@ -88,15 +96,25 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { cassandraCluster = ctx.getCassandraCluster(); if (cassandraCluster == null) { throw new RuntimeException("Unable to connect to Cassandra database"); - } else { - startExecutor(); - saveStmt = getSaveStmt(); + } + ctx.addTenantProfileListener(this::onTenantProfileUpdate); + onTenantProfileUpdate(ctx.getTenantProfile()); + startExecutor(); + saveStmt = getSaveStmt(); + } + + void onTenantProfileUpdate(TenantProfile tenantProfile) { + DefaultTenantProfileConfiguration configuration = (DefaultTenantProfileConfiguration) tenantProfile.getProfileData().getConfiguration(); + long tenantProfileDefaultStorageTtl = TimeUnit.DAYS.toSeconds(configuration.getDefaultStorageTtlDays()); + ttl = config.getDefaultTTL(); + if (ttl == 0L) { + ttl = tenantProfileDefaultStorageTtl; } } @Override public void onMsg(TbContext ctx, TbMsg msg) { - withCallback(save(msg, ctx), aVoid -> ctx.tellSuccess(msg), e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); + withCallback(save(msg, ctx, ttl), aVoid -> ctx.tellSuccess(msg), e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); } @Override @@ -163,10 +181,13 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { query.append("?, "); } } + if (ttl > 0) { + query.append(" USING TTL ?"); + } return query.toString(); } - private ListenableFuture save(TbMsg msg, TbContext ctx) { + private ListenableFuture save(TbMsg msg, TbContext ctx, long ttl) { JsonElement data = JsonParser.parseString(msg.getData()); if (!data.isJsonObject()) { throw new IllegalStateException("Invalid message structure, it is not a JSON Object:" + data); @@ -204,6 +225,9 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } i.getAndIncrement(); }); + if (ttl > 0) { + stmtBuilder.setInt(i.get(), (int) ttl); + } return getFuture(executeAsyncWrite(ctx, stmtBuilder.build()), rs -> null); } } @@ -240,4 +264,20 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { }, readResultsProcessingExecutor); } + @Override + public TbPair upgrade(int fromVersion, JsonNode oldConfiguration) throws TbNodeException { + boolean hasChanges = false; + switch (fromVersion) { + case 0: + if (!oldConfiguration.has("defaultTTL")) { + hasChanges = true; + ((ObjectNode) oldConfiguration).put("defaultTTL", 0); + } + break; + default: + break; + } + return new TbPair<>(hasChanges, oldConfiguration); + } + } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java index 0a5f153192..8d3d78e8b9 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java @@ -27,11 +27,13 @@ public class TbSaveToCustomCassandraTableNodeConfiguration implements NodeConfig private String tableName; private Map fieldsMapping; + private long defaultTTL; @Override public TbSaveToCustomCassandraTableNodeConfiguration defaultConfiguration() { TbSaveToCustomCassandraTableNodeConfiguration configuration = new TbSaveToCustomCassandraTableNodeConfiguration(); + configuration.setDefaultTTL(0L); configuration.setTableName(""); Map map = new HashMap<>(); map.put("", ""); From bef6cfd339b711b9a5e23d5dd39d4c08c760d6e9 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 19 Jun 2024 16:15:08 +0300 Subject: [PATCH 07/37] refactored tests --- .../TbSaveToCustomCassandraTableNode.java | 9 ++-- .../engine/telemetry/TbMsgTimeseriesNode.java | 2 +- .../TbSaveToCustomCassandraTableNodeTest.java | 41 ++++++++++--------- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index 2df4225b0f..d22ffe928a 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -103,18 +103,17 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { saveStmt = getSaveStmt(); } - void onTenantProfileUpdate(TenantProfile tenantProfile) { + private void onTenantProfileUpdate(TenantProfile tenantProfile) { DefaultTenantProfileConfiguration configuration = (DefaultTenantProfileConfiguration) tenantProfile.getProfileData().getConfiguration(); - long tenantProfileDefaultStorageTtl = TimeUnit.DAYS.toSeconds(configuration.getDefaultStorageTtlDays()); ttl = config.getDefaultTTL(); if (ttl == 0L) { - ttl = tenantProfileDefaultStorageTtl; + ttl = TimeUnit.DAYS.toSeconds(configuration.getDefaultStorageTtlDays()); } } @Override public void onMsg(TbContext ctx, TbMsg msg) { - withCallback(save(msg, ctx, ttl), aVoid -> ctx.tellSuccess(msg), e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); + withCallback(save(msg, ctx), aVoid -> ctx.tellSuccess(msg), e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); } @Override @@ -187,7 +186,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { return query.toString(); } - private ListenableFuture save(TbMsg msg, TbContext ctx, long ttl) { + private ListenableFuture save(TbMsg msg, TbContext ctx) { JsonElement data = JsonParser.parseString(msg.getData()); if (!data.isJsonObject()) { throw new IllegalStateException("Invalid message structure, it is not a JSON Object: " + data); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 081804639a..97b2ac116f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -75,7 +75,7 @@ public class TbMsgTimeseriesNode implements TbNode { onTenantProfileUpdate(ctx.getTenantProfile()); } - void onTenantProfileUpdate(TenantProfile tenantProfile) { + private void onTenantProfileUpdate(TenantProfile tenantProfile) { DefaultTenantProfileConfiguration configuration = (DefaultTenantProfileConfiguration) tenantProfile.getProfileData().getConfiguration(); tenantProfileDefaultStorageTtl = TimeUnit.DAYS.toSeconds(configuration.getDefaultStorageTtlDays()); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index 3ab997238a..5d520d5ab9 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -57,6 +57,7 @@ import org.thingsboard.server.dao.nosql.CassandraStatementTask; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.nosql.TbResultSetFuture; +import java.util.LinkedHashMap; import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -75,10 +76,10 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.never; import static org.mockito.BDDMockito.spy; import static org.mockito.BDDMockito.then; import static org.mockito.BDDMockito.willAnswer; -import static org.mockito.BDDMockito.never; @ExtendWith(MockitoExtension.class) public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgradeTest { @@ -136,7 +137,9 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); assertThatThrownBy(() -> node.init(ctxMock, configuration)) .isInstanceOf(TbNodeException.class) - .hasMessage("Unable to connect to Cassandra database"); + .hasMessage("Unable to connect to Cassandra database") + .extracting(e -> ((TbNodeException) e).isUnrecoverable()) + .isEqualTo(false); } @Test @@ -238,15 +241,15 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad return Stream.of( Arguments.of(0, 0, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?)", (Consumer) builder -> { - then(builder).should(never()).setInt(anyInt(), eq(0)); + then(builder).should(never()).setInt(anyInt(), anyInt()); }), Arguments.of(0, 5, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?) USING TTL ?", (Consumer) builder -> { - then(builder).should().setInt(anyInt(), eq(432000)); + then(builder).should().setInt(1, 432000); }), Arguments.of(20, 1, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?) USING TTL ?", (Consumer) builder -> { - then(builder).should().setInt(anyInt(), eq(20)); + then(builder).should().setInt(1, 20); }) ); } @@ -255,14 +258,14 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad public void givenValidMsgStructure_whenOnMsg_thenSaveToCustomCassandraTable() throws TbNodeException { config.setDefaultTTL(25L); config.setTableName("readings"); - config.setFieldsMapping(Map.of( - "$entityId", "entityIdTableColumn", - "doubleField", "doubleTableColumn", - "longField", "longTableColumn", - "booleanField", "booleanTableColumn", - "stringField", "stringTableColumn", - "jsonField", "jsonTableColumn" - )); + Map mappings = new LinkedHashMap<>(); + mappings.put("$entityId", "entityIdTableColumn"); + mappings.put("doubleField", "doubleTableColumn"); + mappings.put("longField", "longTableColumn"); + mappings.put("booleanField", "booleanTableColumn"); + mappings.put("stringField", "stringTableColumn"); + mappings.put("jsonField", "jsonTableColumn"); + config.setFieldsMapping(mappings); mockOnInit(); mockBoundStatementBuilder(); @@ -350,12 +353,12 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad } private void verifySettingStatementBuilder() { - then(boundStatementBuilderMock).should().setUuid(anyInt(), eq(DEVICE_ID.getId())); - then(boundStatementBuilderMock).should().setDouble(anyInt(), eq(22.5)); - then(boundStatementBuilderMock).should().setLong(anyInt(), eq(56L)); - then(boundStatementBuilderMock).should().setBoolean(anyInt(), eq(true)); - then(boundStatementBuilderMock).should().setString(anyInt(), eq("some string")); - then(boundStatementBuilderMock).should().setString(anyInt(), eq("{\"key\":\"value\"}")); + then(boundStatementBuilderMock).should().setUuid(0, DEVICE_ID.getId()); + then(boundStatementBuilderMock).should().setDouble(1, 22.5); + then(boundStatementBuilderMock).should().setLong(2, 56L); + then(boundStatementBuilderMock).should().setBoolean(3, true); + then(boundStatementBuilderMock).should().setString(4, "some string"); + then(boundStatementBuilderMock).should().setString(5, "{\"key\":\"value\"}"); then(boundStatementBuilderMock).should().setInt(anyInt(), eq(25)); } From bfdea4d7e3acac12c9b314b697b44dda3e560b08 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 20 Jun 2024 10:34:20 +0300 Subject: [PATCH 08/37] made exception in init method unrecoverable --- .../rule/engine/action/TbSaveToCustomCassandraTableNode.java | 2 +- .../engine/action/TbSaveToCustomCassandraTableNodeTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index d22ffe928a..e1aa1f6a13 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -95,7 +95,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { config = TbNodeUtils.convert(configuration, TbSaveToCustomCassandraTableNodeConfiguration.class); cassandraCluster = ctx.getCassandraCluster(); if (cassandraCluster == null) { - throw new TbNodeException("Unable to connect to Cassandra database"); + throw new TbNodeException("Unable to connect to Cassandra database", true); } ctx.addTenantProfileListener(this::onTenantProfileUpdate); onTenantProfileUpdate(ctx.getTenantProfile()); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index 5d520d5ab9..27d69f2d66 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -139,7 +139,7 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad .isInstanceOf(TbNodeException.class) .hasMessage("Unable to connect to Cassandra database") .extracting(e -> ((TbNodeException) e).isUnrecoverable()) - .isEqualTo(false); + .isEqualTo(true); } @Test From 88a768f06732c46987fcd462ab8d8c0dc23c8581 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 21 Jun 2024 10:50:40 +0300 Subject: [PATCH 09/37] added verification for match values insertion into statement --- .../TbSaveToCustomCassandraTableNodeTest.java | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index 27d69f2d66..b3f3f2caa4 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -34,6 +34,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.AbstractRuleNodeUpgradeTest; @@ -57,7 +58,8 @@ import org.thingsboard.server.dao.nosql.CassandraStatementTask; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.nosql.TbResultSetFuture; -import java.util.LinkedHashMap; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -74,7 +76,6 @@ import static org.mockito.ArgumentMatchers.anyDouble; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.BDDMockito.given; import static org.mockito.BDDMockito.never; import static org.mockito.BDDMockito.spy; @@ -255,16 +256,17 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad } @Test - public void givenValidMsgStructure_whenOnMsg_thenSaveToCustomCassandraTable() throws TbNodeException { + public void givenValidMsgStructure_whenOnMsg_thenVerifyMatchOfValuesInsertionOrderIntoStatementAndSaveToCustomCassandraTable() throws TbNodeException { config.setDefaultTTL(25L); config.setTableName("readings"); - Map mappings = new LinkedHashMap<>(); - mappings.put("$entityId", "entityIdTableColumn"); - mappings.put("doubleField", "doubleTableColumn"); - mappings.put("longField", "longTableColumn"); - mappings.put("booleanField", "booleanTableColumn"); - mappings.put("stringField", "stringTableColumn"); - mappings.put("jsonField", "jsonTableColumn"); + Map mappings = Map.of( + "$entityId", "entityIdTableColumn", + "doubleField", "doubleTableColumn", + "longField", "longTableColumn", + "booleanField", "booleanTableColumn", + "stringField", "stringTableColumn", + "jsonField", "jsonTableColumn" + ); config.setFieldsMapping(mappings); mockOnInit(); @@ -353,13 +355,15 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad } private void verifySettingStatementBuilder() { - then(boundStatementBuilderMock).should().setUuid(0, DEVICE_ID.getId()); - then(boundStatementBuilderMock).should().setDouble(1, 22.5); - then(boundStatementBuilderMock).should().setLong(2, 56L); - then(boundStatementBuilderMock).should().setBoolean(3, true); - then(boundStatementBuilderMock).should().setString(4, "some string"); - then(boundStatementBuilderMock).should().setString(5, "{\"key\":\"value\"}"); - then(boundStatementBuilderMock).should().setInt(anyInt(), eq(25)); + Map fieldsMap = (Map) ReflectionTestUtils.getField(node, "fieldsMap"); + List values = new ArrayList<>(fieldsMap.values()); + then(boundStatementBuilderMock).should().setUuid(values.indexOf("entityIdTableColumn"), DEVICE_ID.getId()); + then(boundStatementBuilderMock).should().setDouble(values.indexOf("doubleTableColumn"), 22.5); + then(boundStatementBuilderMock).should().setLong(values.indexOf("longTableColumn"), 56L); + then(boundStatementBuilderMock).should().setBoolean(values.indexOf("booleanTableColumn"), true); + then(boundStatementBuilderMock).should().setString(values.indexOf("stringTableColumn"), "some string"); + then(boundStatementBuilderMock).should().setString(values.indexOf("jsonTableColumn"), "{\"key\":\"value\"}"); + then(boundStatementBuilderMock).should().setInt(values.size(), 25); } private TenantProfile getTenantProfileWithTtl(int ttlInDays) { From 4307d6f13ae2e8ca9887ebe95f468dd1f22d6828 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 1 Jul 2024 10:22:40 +0300 Subject: [PATCH 10/37] fixed rewriting ttl for existing nodes --- .../TbSaveToCustomCassandraTableNode.java | 28 +++++++--- ...CustomCassandraTableNodeConfiguration.java | 4 +- .../TbSaveToCustomCassandraTableNodeTest.java | 56 +++++++++++++++++-- 3 files changed, 73 insertions(+), 15 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index e1aa1f6a13..fcaf2420e9 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -21,6 +21,8 @@ import com.datastax.oss.driver.api.core.cql.BoundStatement; import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.datastax.oss.driver.api.core.cql.PreparedStatement; import com.datastax.oss.driver.api.core.cql.Statement; +import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; +import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Function; @@ -88,7 +90,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { private PreparedStatement saveStmt; private ExecutorService readResultsProcessingExecutor; private Map fieldsMap; - private long ttl; + private Integer ttl; @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { @@ -97,6 +99,9 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { if (cassandraCluster == null) { throw new TbNodeException("Unable to connect to Cassandra database", true); } + if (!isTableExists()) { + throw new TbNodeException("Table '" + TABLE_PREFIX + config.getTableName() + "' does not exist in Cassandra cluster.", true); + } ctx.addTenantProfileListener(this::onTenantProfileUpdate); onTenantProfileUpdate(ctx.getTenantProfile()); startExecutor(); @@ -106,8 +111,8 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { private void onTenantProfileUpdate(TenantProfile tenantProfile) { DefaultTenantProfileConfiguration configuration = (DefaultTenantProfileConfiguration) tenantProfile.getProfileData().getConfiguration(); ttl = config.getDefaultTTL(); - if (ttl == 0L) { - ttl = TimeUnit.DAYS.toSeconds(configuration.getDefaultStorageTtlDays()); + if (ttl != null && ttl == 0) { + ttl = (int) TimeUnit.DAYS.toSeconds(configuration.getDefaultStorageTtlDays()); } } @@ -132,6 +137,15 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } } + private boolean isTableExists() { + KeyspaceMetadata keyspaceMetadata = getSession().getMetadata().getKeyspace(cassandraCluster.getKeyspaceName()).orElse(null); + if (keyspaceMetadata != null) { + TableMetadata tableMetadata = keyspaceMetadata.getTable(TABLE_PREFIX + config.getTableName()).orElse(null); + return tableMetadata != null; + } + return false; + } + private PreparedStatement prepare(String query) { return getSession().prepare(query); } @@ -180,7 +194,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { query.append("?, "); } } - if (ttl > 0) { + if (ttl != null && ttl > 0) { query.append(" USING TTL ?"); } return query.toString(); @@ -224,8 +238,8 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } i.getAndIncrement(); }); - if (ttl > 0) { - stmtBuilder.setInt(i.get(), (int) ttl); + if (ttl != null && ttl > 0) { + stmtBuilder.setInt(i.get(), ttl); } return getFuture(executeAsyncWrite(ctx, stmtBuilder.build()), rs -> null); } @@ -274,7 +288,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { case 0: if (!oldConfiguration.has("defaultTTL")) { hasChanges = true; - ((ObjectNode) oldConfiguration).put("defaultTTL", 0); + ((ObjectNode) oldConfiguration).putNull("defaultTTL"); } break; default: diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java index 8d3d78e8b9..3efae275da 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java @@ -27,13 +27,13 @@ public class TbSaveToCustomCassandraTableNodeConfiguration implements NodeConfig private String tableName; private Map fieldsMapping; - private long defaultTTL; + private Integer defaultTTL; @Override public TbSaveToCustomCassandraTableNodeConfiguration defaultConfiguration() { TbSaveToCustomCassandraTableNodeConfiguration configuration = new TbSaveToCustomCassandraTableNodeConfiguration(); - configuration.setDefaultTTL(0L); + configuration.setDefaultTTL(0); configuration.setTableName(""); Map map = new HashMap<>(); map.put("", ""); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index b3f3f2caa4..d691f1eaa4 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -21,7 +21,10 @@ import com.datastax.oss.driver.api.core.cql.BoundStatement; import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.datastax.oss.driver.api.core.cql.ColumnDefinitions; import com.datastax.oss.driver.api.core.cql.PreparedStatement; +import com.datastax.oss.driver.api.core.metadata.Metadata; import com.datastax.oss.driver.api.core.metadata.Node; +import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; +import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; import com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry; import com.google.common.util.concurrent.SettableFuture; import org.junit.jupiter.api.AfterEach; @@ -61,6 +64,7 @@ import org.thingsboard.server.dao.nosql.TbResultSetFuture; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -114,6 +118,12 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad private ProtocolVersion protocolVersionMock; @Mock private Node nodeMock; + @Mock + private Metadata metadataMock; + @Mock + private KeyspaceMetadata keyspaceMetadataMock; + @Mock + private TableMetadata tableMetadataMock; @BeforeEach public void setUp() { @@ -130,7 +140,7 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad public void verifyDefaultConfig() { assertThat(config.getTableName()).isEqualTo(""); assertThat(config.getFieldsMapping()).isEqualTo(Map.of("", "")); - assertThat(config.getDefaultTTL()).isEqualTo(0L); + assertThat(config.getDefaultTTL()).isEqualTo(0); } @Test @@ -143,12 +153,28 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad .isEqualTo(true); } + @Test + public void givenTableDoesNotExist_whenInit_thenThrowsException() { + config.setTableName("test_table"); + var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); + + mockCassandraCluster(); + given(keyspaceMetadataMock.getTable(anyString())).willReturn(Optional.empty()); + + assertThatThrownBy(() -> node.init(ctxMock, configuration)) + .isInstanceOf(TbNodeException.class) + .hasMessage("Table 'cs_tb_test_table' does not exist in Cassandra cluster.") + .extracting(e -> ((TbNodeException) e).isUnrecoverable()) + .isEqualTo(true); + } + @Test public void givenFieldsMapIsEmpty_whenInit_thenThrowsException() { + config.setTableName("test_table"); config.setFieldsMapping(emptyMap()); var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); - given(ctxMock.getCassandraCluster()).willReturn(cassandraClusterMock); + mockCassandraCluster(); given(ctxMock.getTenantProfile()).willReturn(getTenantProfileWithTtl(5)); assertThatThrownBy(() -> node.init(ctxMock, configuration)) @@ -215,7 +241,7 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad @ParameterizedTest @MethodSource - public void givenTtl_whenOnMsg_thenVerifyStatement(long ttlFromConfig, + public void givenTtl_whenOnMsg_thenVerifyStatement(Integer ttlFromConfig, int ttlFromTenantProfileInDays, String expectedQuery, Consumer verifyBuilder) throws TbNodeException { @@ -251,13 +277,17 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad Arguments.of(20, 1, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?) USING TTL ?", (Consumer) builder -> { then(builder).should().setInt(1, 20); + }), + Arguments.of(null, 2, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?)", + (Consumer) builder -> { + then(builder).should(never()).setInt(anyInt(), anyInt()); }) ); } @Test public void givenValidMsgStructure_whenOnMsg_thenVerifyMatchOfValuesInsertionOrderIntoStatementAndSaveToCustomCassandraTable() throws TbNodeException { - config.setDefaultTTL(25L); + config.setDefaultTTL(25); config.setTableName("readings"); Map mappings = Map.of( "$entityId", "entityIdTableColumn", @@ -311,19 +341,33 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad Arguments.of(0, "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"}}", true, + "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTTL\":null}" + ), + // default config for version 1 with upgrade from version 1 + Arguments.of(1, + "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTTL\":0}", + false, "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTTL\":0}" ) ); } private void mockOnInit() { - given(ctxMock.getCassandraCluster()).willReturn(cassandraClusterMock); + mockCassandraCluster(); given(ctxMock.getTenantProfile()).willReturn(getTenantProfileWithTtl(5)); - given(cassandraClusterMock.getSession()).willReturn(sessionMock); given(cassandraClusterMock.getDefaultWriteConsistencyLevel()).willReturn(DefaultConsistencyLevel.ONE); given(sessionMock.prepare(anyString())).willReturn(preparedStatementMock); } + private void mockCassandraCluster() { + given(ctxMock.getCassandraCluster()).willReturn(cassandraClusterMock); + given(cassandraClusterMock.getSession()).willReturn(sessionMock); + given(sessionMock.getMetadata()).willReturn(metadataMock); + given(cassandraClusterMock.getKeyspaceName()).willReturn("test_keyspace"); + given(metadataMock.getKeyspace(anyString())).willReturn(Optional.of(keyspaceMetadataMock)); + given(keyspaceMetadataMock.getTable(anyString())).willReturn(Optional.of(tableMetadataMock)); + } + private void mockBoundStatement() { given(preparedStatementMock.bind()).willReturn(boundStatementMock); given(boundStatementMock.getPreparedStatement()).willReturn(preparedStatementMock); From 47afd13cf7af100cb14d7ad72bc9aaaed6201ac3 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 1 Jul 2024 12:17:47 +0300 Subject: [PATCH 11/37] made exception on init recoverable --- .../action/TbSaveToCustomCassandraTableNode.java | 13 ++++--------- .../TbSaveToCustomCassandraTableNodeTest.java | 2 +- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index fcaf2420e9..ddc4357674 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -21,8 +21,6 @@ import com.datastax.oss.driver.api.core.cql.BoundStatement; import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.datastax.oss.driver.api.core.cql.PreparedStatement; import com.datastax.oss.driver.api.core.cql.Statement; -import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; -import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Function; @@ -100,7 +98,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { throw new TbNodeException("Unable to connect to Cassandra database", true); } if (!isTableExists()) { - throw new TbNodeException("Table '" + TABLE_PREFIX + config.getTableName() + "' does not exist in Cassandra cluster.", true); + throw new TbNodeException("Table '" + TABLE_PREFIX + config.getTableName() + "' does not exist in Cassandra cluster."); } ctx.addTenantProfileListener(this::onTenantProfileUpdate); onTenantProfileUpdate(ctx.getTenantProfile()); @@ -138,12 +136,9 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } private boolean isTableExists() { - KeyspaceMetadata keyspaceMetadata = getSession().getMetadata().getKeyspace(cassandraCluster.getKeyspaceName()).orElse(null); - if (keyspaceMetadata != null) { - TableMetadata tableMetadata = keyspaceMetadata.getTable(TABLE_PREFIX + config.getTableName()).orElse(null); - return tableMetadata != null; - } - return false; + var keyspaceMdOpt = getSession().getMetadata().getKeyspace(cassandraCluster.getKeyspaceName()); + return keyspaceMdOpt.map(keyspaceMetadata -> + keyspaceMetadata.getTable(TABLE_PREFIX + config.getTableName()).isPresent()).orElse(false); } private PreparedStatement prepare(String query) { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index d691f1eaa4..f3e7d0d004 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -165,7 +165,7 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad .isInstanceOf(TbNodeException.class) .hasMessage("Table 'cs_tb_test_table' does not exist in Cassandra cluster.") .extracting(e -> ((TbNodeException) e).isUnrecoverable()) - .isEqualTo(true); + .isEqualTo(false); } @Test From d3d20aa449b0ff7103a55bda971e66e0bc9a7d9f Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 18 Oct 2024 16:45:11 +0200 Subject: [PATCH 12/37] used enqueue for input node for avoid latency --- .../server/actors/ruleChain/DefaultTbContext.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index ec10402821..b5b389da78 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -172,8 +172,18 @@ class DefaultTbContext implements TbContext { @Override public void input(TbMsg msg, RuleChainId ruleChainId) { + ack(msg); + if (!msg.isValid()) { + return; + } msg.pushToStack(nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId()); - nodeCtx.getChainActor().tell(new RuleChainInputMsg(ruleChainId, msg)); + TbMsg inputMsg = msg.copyWithRuleChainId(ruleChainId); + TransportProtos.ToRuleEngineMsg toReMsg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(inputMsg)).build(); + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getQueueName(), getTenantId(), inputMsg.getOriginator()); + mainCtx.getClusterService().pushMsgToRuleEngine(tpi, inputMsg.getId(), toReMsg, null); } @Override From e2a356703f5c4b7e8909d182f210fb102cd1d246 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 18 Oct 2024 17:05:22 +0200 Subject: [PATCH 13/37] minor refactoring, ack used if push is success --- .../actors/ruleChain/DefaultTbContext.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index b5b389da78..23741ad965 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -109,6 +109,8 @@ import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.common.SimpleTbQueueCallback; import org.thingsboard.server.service.executors.PubSubRuleNodeExecutorProvider; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; @@ -172,18 +174,27 @@ class DefaultTbContext implements TbContext { @Override public void input(TbMsg msg, RuleChainId ruleChainId) { - ack(msg); if (!msg.isValid()) { return; } - msg.pushToStack(nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId()); TbMsg inputMsg = msg.copyWithRuleChainId(ruleChainId); + inputMsg.pushToStack(nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId()); TransportProtos.ToRuleEngineMsg toReMsg = TransportProtos.ToRuleEngineMsg.newBuilder() .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) .setTbMsg(TbMsg.toByteString(inputMsg)).build(); TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getQueueName(), getTenantId(), inputMsg.getOriginator()); - mainCtx.getClusterService().pushMsgToRuleEngine(tpi, inputMsg.getId(), toReMsg, null); + mainCtx.getClusterService().pushMsgToRuleEngine(tpi, inputMsg.getId(), toReMsg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + ack(msg); + } + + @Override + public void onFailure(Throwable error) { + tellFailure(msg, error); + } + }); } @Override From 726c95c661516115171bbf5e1a18e3fc05d60dce Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Sun, 20 Oct 2024 13:41:56 +0200 Subject: [PATCH 14/37] fixed tests --- .../flow/AbstractRuleEngineFlowIntegrationTest.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 9e36fed677..920ffdd211 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.rules.flow; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -59,6 +60,7 @@ import org.thingsboard.server.dao.event.EventService; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.mockito.Mockito.spy; @@ -331,6 +333,15 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule RuleChain finalRuleChain = rootRuleChain; RuleNode lastRuleNode = secondaryMetaData.getNodes().stream().filter(node -> !node.getId().equals(finalRuleChain.getFirstRuleNodeId())).findFirst().get(); + Awaitility.await().atMost(3, TimeUnit.SECONDS) + .until(() -> + getDebugEvents(savedTenant.getId(), lastRuleNode.getId(), 1000) + .getData() + .stream() + .filter(filterByPostTelemetryEventType()) + .count() == 2 + ); + eventsPage = getDebugEvents(savedTenant.getId(), lastRuleNode.getId(), 1000); events = eventsPage.getData().stream().filter(filterByPostTelemetryEventType()).collect(Collectors.toList()); From 9ba71056cebff1976d92e032abe15c35d46b704a Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 25 Oct 2024 11:52:54 +0300 Subject: [PATCH 15/37] refactored UserService to reuse functionality --- .../service/entitiy/user/DefaultUserService.java | 10 ++-------- .../org/thingsboard/server/dao/user/UserService.java | 2 ++ .../thingsboard/server/dao/user/UserServiceImpl.java | 10 ++++++++++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/user/DefaultUserService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/user/DefaultUserService.java index ad4b7c097e..3fe8f27873 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/user/DefaultUserService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/user/DefaultUserService.java @@ -90,16 +90,10 @@ public class DefaultUserService extends AbstractTbEntityService implements TbUse public UserActivationLink getActivationLink(TenantId tenantId, CustomerId customerId, UserId userId, HttpServletRequest request) throws ThingsboardException { UserCredentials userCredentials = userService.findUserCredentialsByUserId(tenantId, userId); if (!userCredentials.isEnabled() && userCredentials.getActivateToken() != null) { - long ttl = userCredentials.getActivationTokenTtl(); - if (ttl < TimeUnit.MINUTES.toMillis(15)) { // renew link if less than 15 minutes before expiration - userCredentials = userService.generateUserActivationToken(userCredentials); - userCredentials = userService.saveUserCredentials(tenantId, userCredentials); - ttl = userCredentials.getActivationTokenTtl(); - log.debug("[{}][{}] Regenerated expired user activation token", tenantId, userId); - } + userCredentials = userService.refreshUserActivationToken(tenantId, userCredentials); String baseUrl = systemSecurityService.getBaseUrl(tenantId, customerId, request); String link = baseUrl + "/api/noauth/activate?activateToken=" + userCredentials.getActivateToken(); - return new UserActivationLink(link, ttl); + return new UserActivationLink(link, userCredentials.getActivationTokenTtl()); } else { throw new ThingsboardException("User is already activated!", ThingsboardErrorCode.BAD_REQUEST_PARAMS); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java index 8f22812cfc..0d532cdfac 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java @@ -63,6 +63,8 @@ public interface UserService extends EntityDaoService { UserCredentials generateUserActivationToken(UserCredentials userCredentials); + UserCredentials refreshUserActivationToken(TenantId tenantId, UserCredentials userCredentials); + UserCredentials replaceUserCredentials(TenantId tenantId, UserCredentials userCredentials); void deleteUser(TenantId tenantId, User user); diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index ba0d3d69f3..d8eee2967a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -300,6 +300,16 @@ public class UserServiceImpl extends AbstractCachedEntityService Date: Wed, 6 Nov 2024 15:24:02 +0100 Subject: [PATCH 16/37] minor refactoring due to comments --- .../actors/ruleChain/DefaultTbContext.java | 37 +++++++------------ ...AbstractRuleEngineFlowIntegrationTest.java | 2 +- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 23741ad965..5b4902afa5 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -110,7 +110,6 @@ import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.common.SimpleTbQueueCallback; import org.thingsboard.server.service.executors.PubSubRuleNodeExecutorProvider; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; @@ -177,24 +176,10 @@ class DefaultTbContext implements TbContext { if (!msg.isValid()) { return; } - TbMsg inputMsg = msg.copyWithRuleChainId(ruleChainId); - inputMsg.pushToStack(nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId()); - TransportProtos.ToRuleEngineMsg toReMsg = TransportProtos.ToRuleEngineMsg.newBuilder() - .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(inputMsg)).build(); - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getQueueName(), getTenantId(), inputMsg.getOriginator()); - mainCtx.getClusterService().pushMsgToRuleEngine(tpi, inputMsg.getId(), toReMsg, new TbQueueCallback() { - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - ack(msg); - } - - @Override - public void onFailure(Throwable error) { - tellFailure(msg, error); - } - }); + TbMsg tbMsg = msg.copyWithRuleChainId(ruleChainId); + tbMsg.pushToStack(nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId()); + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getQueueName(), getTenantId(), tbMsg.getOriginator()); + doEnqueue(tpi, tbMsg, new SimpleTbQueueCallback(md -> ack(msg), t -> tellFailure(msg, t))); } @Override @@ -230,14 +215,10 @@ class DefaultTbContext implements TbContext { } return; } - TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() - .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(tbMsg)).build(); if (nodeCtx.getSelf().isDebugMode()) { mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), tbMsg, "To Root Rule Chain"); } - mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, new SimpleTbQueueCallback( + doEnqueue(tpi, tbMsg, new SimpleTbQueueCallback( metadata -> { if (onSuccess != null) { onSuccess.run(); @@ -252,6 +233,14 @@ class DefaultTbContext implements TbContext { })); } + private void doEnqueue(TopicPartitionInfo tpi, TbMsg tbMsg, TbQueueCallback callback) { + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, callback); + } + @Override public void enqueueForTellFailure(TbMsg tbMsg, String failureMessage) { TopicPartitionInfo tpi = resolvePartition(tbMsg); diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 920ffdd211..143c6c72a1 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -333,7 +333,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule RuleChain finalRuleChain = rootRuleChain; RuleNode lastRuleNode = secondaryMetaData.getNodes().stream().filter(node -> !node.getId().equals(finalRuleChain.getFirstRuleNodeId())).findFirst().get(); - Awaitility.await().atMost(3, TimeUnit.SECONDS) + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getDebugEvents(savedTenant.getId(), lastRuleNode.getId(), 1000) .getData() From 5535f11f224473de1326768a919b486a5558129f Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 12 Nov 2024 11:51:15 +0200 Subject: [PATCH 17/37] fixed log printing --- .../server/service/entitiy/user/DefaultUserService.java | 4 +--- .../java/org/thingsboard/server/dao/user/UserService.java | 2 +- .../java/org/thingsboard/server/dao/user/UserServiceImpl.java | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/user/DefaultUserService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/user/DefaultUserService.java index 3fe8f27873..f98f05b447 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/user/DefaultUserService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/user/DefaultUserService.java @@ -35,8 +35,6 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import org.thingsboard.server.service.security.system.SystemSecurityService; -import java.util.concurrent.TimeUnit; - @Service @TbCoreComponent @AllArgsConstructor @@ -90,7 +88,7 @@ public class DefaultUserService extends AbstractTbEntityService implements TbUse public UserActivationLink getActivationLink(TenantId tenantId, CustomerId customerId, UserId userId, HttpServletRequest request) throws ThingsboardException { UserCredentials userCredentials = userService.findUserCredentialsByUserId(tenantId, userId); if (!userCredentials.isEnabled() && userCredentials.getActivateToken() != null) { - userCredentials = userService.refreshUserActivationToken(tenantId, userCredentials); + userCredentials = userService.checkUserActivationToken(tenantId, userCredentials); String baseUrl = systemSecurityService.getBaseUrl(tenantId, customerId, request); String link = baseUrl + "/api/noauth/activate?activateToken=" + userCredentials.getActivateToken(); return new UserActivationLink(link, userCredentials.getActivationTokenTtl()); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java index 0d532cdfac..a33d64b746 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java @@ -63,7 +63,7 @@ public interface UserService extends EntityDaoService { UserCredentials generateUserActivationToken(UserCredentials userCredentials); - UserCredentials refreshUserActivationToken(TenantId tenantId, UserCredentials userCredentials); + UserCredentials checkUserActivationToken(TenantId tenantId, UserCredentials userCredentials); UserCredentials replaceUserCredentials(TenantId tenantId, UserCredentials userCredentials); diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index d8eee2967a..16b25a7ede 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -301,12 +301,12 @@ public class UserServiceImpl extends AbstractCachedEntityService Date: Tue, 19 Nov 2024 09:43:14 +0200 Subject: [PATCH 18/37] Fixed non returning SubAck for provisioning clients --- .../transport/mqtt/MqttTransportHandler.java | 23 +++++++++- .../msa/connectivity/MqttClientTest.java | 43 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index d7f64e58ee..6d5c0f5d50 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -313,6 +313,28 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement sendResponseForAdaptorErrorOrCloseContext(ctx, topicName, msgId); } break; + case SUBSCRIBE: + MqttSubscribeMessage mqttSubscribeMsg = (MqttSubscribeMessage) msg; + log.trace("[{}] Processing subscription [{}]!", sessionId, mqttSubscribeMsg.variableHeader().messageId()); + List grantedQoSList = new ArrayList<>(); + for (MqttTopicSubscription subscription : mqttSubscribeMsg.payload().topicSubscriptions()) { + String topic = subscription.topicFilter(); + MqttQoS reqQoS = subscription.qualityOfService(); + try { + if (MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC.equals(topic)) { + registerSubQoS(topic, grantedQoSList, reqQoS); + } else { + log.warn("[{}] Failed to subscribe to [{}][{}]", sessionId, topic, reqQoS); + grantedQoSList.add(ReturnCodeResolver.getSubscriptionReturnCode(deviceSessionCtx.getMqttVersion(), MqttReasonCodes.SubAck.TOPIC_FILTER_INVALID)); + break; + } + } catch (Exception e) { + log.warn("[{}] Failed to subscribe to [{}][{}]", sessionId, topic, reqQoS, e); + grantedQoSList.add(ReturnCodeResolver.getSubscriptionReturnCode(deviceSessionCtx.getMqttVersion(), MqttReasonCodes.SubAck.IMPLEMENTATION_SPECIFIC_ERROR)); + } + } + ctx.writeAndFlush(createSubAckMessage(mqttSubscribeMsg.variableHeader().messageId(), grantedQoSList)); + break; case PINGREQ: ctx.writeAndFlush(new MqttMessage(new MqttFixedHeader(PINGRESP, false, AT_MOST_ONCE, false, 0))); break; @@ -822,7 +844,6 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement case MqttTopics.GATEWAY_ATTRIBUTES_TOPIC: case MqttTopics.GATEWAY_RPC_TOPIC: case MqttTopics.GATEWAY_ATTRIBUTES_RESPONSE_TOPIC: - case MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC: case MqttTopics.DEVICE_FIRMWARE_RESPONSES_TOPIC: case MqttTopics.DEVICE_FIRMWARE_ERROR_TOPIC: case MqttTopics.DEVICE_SOFTWARE_RESPONSES_TOPIC: diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java index c298d4fea3..8c26be9b45 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java @@ -28,6 +28,7 @@ import io.netty.handler.codec.mqtt.MqttMessage; import io.netty.handler.codec.mqtt.MqttQoS; import io.netty.handler.codec.mqtt.MqttReasonCodeAndPropertiesVariableHeader; import io.netty.handler.codec.mqtt.MqttReasonCodes; +import io.netty.handler.codec.mqtt.MqttSubAckMessage; import io.netty.handler.codec.mqtt.MqttVersion; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -491,6 +492,48 @@ public class MqttClientTest extends AbstractContainerTest { updateDeviceProfileWithProvisioningStrategy(deviceProfile, DeviceProfileProvisionType.DISABLED); } + @Test + public void provisionRequestForCheckSubAckReceived() throws Exception { + + DeviceProfile deviceProfile = testRestClient.getDeviceProfileById(device.getDeviceProfileId()); + deviceProfile = updateDeviceProfileWithProvisioningStrategy(deviceProfile, DeviceProfileProvisionType.ALLOW_CREATE_NEW_DEVICES); + + MqttMessageListener listener = new MqttMessageListener(); + MqttClient mqttClient = getMqttClient("provision", listener, MqttVersion.MQTT_5); + final MqttReasonCodes.SubAck[] subAckResult = new MqttReasonCodes.SubAck[1]; + mqttClient.setCallback(new MqttClientCallback() { + @Override + public void connectionLost(Throwable cause) { + } + + @Override + public void onSuccessfulReconnect() { + } + + @Override + public void onSubAck(MqttSubAckMessage subAckMessage) { + subAckResult[0] = subAckMessage.payload().typedReasonCodes().get(0); + } + } + ); + + mqttClient.on("/provision/response", listener, MqttQoS.AT_LEAST_ONCE).get(3 * timeoutMultiplier, TimeUnit.SECONDS); + TimeUnit.SECONDS.sleep(2 * timeoutMultiplier); + assertThat(subAckResult[0]).isNotNull(); + assertThat(MqttReasonCodes.SubAck.GRANTED_QOS_1.equals(subAckResult[0])); + + subAckResult[0] = null; + mqttClient.on("v1/devices/me/attributes", listener, MqttQoS.AT_LEAST_ONCE).get(3 * timeoutMultiplier, TimeUnit.SECONDS); + TimeUnit.SECONDS.sleep(2 * timeoutMultiplier); + assertThat(subAckResult[0]).isNotNull(); + assertThat(MqttReasonCodes.SubAck.TOPIC_FILTER_INVALID.equals(subAckResult[0])); + + testRestClient.deleteDeviceIfExists(device.getId()); + updateDeviceProfileWithProvisioningStrategy(deviceProfile, DeviceProfileProvisionType.DISABLED); + } + + + @Test public void provisionRequestForDeviceWithDisabledProvisioningStrategy() throws Exception { From b0c89f2931b9362926053ba397ab439df8041aef Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 20 Nov 2024 19:09:02 +0100 Subject: [PATCH 19/37] Entity Data Query optimization - entityFilter.isFetchLastLevelOnly() to hit index and avoid seq scan. (and nr.relation_type_group = 'COMMON') --- .../server/dao/sql/query/DefaultEntityQueryRepository.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index dd1b4278d7..36ae779edd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -663,7 +663,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { .append("nr.").append(fromOrTo).append("_id").append(" = re.").append(toOrFrom).append("_id") .append(" and ") .append("nr.").append(fromOrTo).append("_type").append(" = re.").append(toOrFrom).append("_type"); - + notExistsPart.append(" and nr.relation_type_group = 'COMMON'"); // hit the index, the same condition are on the recursive query notExistsPart.append(")"); whereFilter += " and ( r_int.lvl = " + entityFilter.getMaxLevel() + " OR " + notExistsPart.toString() + ")"; } @@ -755,7 +755,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { .append("nr.").append(fromOrTo).append("_type").append(" = re.").append(toOrFrom).append("_type") .append(" and ") .append(whereFilter.toString().replaceAll("re\\.", "nr\\.")); - + notExistsPart.append(" and nr.relation_type_group = 'COMMON'"); // hit the index, the same condition are on the recursive query notExistsPart.append(")"); whereFilter.append(" and ( r_int.lvl = ").append(entityFilter.getMaxLevel()).append(" OR ").append(notExistsPart.toString()).append(")"); } From 1397ce80a02ee85794957ac8e39b332dff297cad Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 8 Nov 2024 08:36:09 +0100 Subject: [PATCH 20/37] set bigger timeout for getting resutl in tests --- .../timeseries/sql/LatestTimeseriesPerformanceTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/sql/LatestTimeseriesPerformanceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/sql/LatestTimeseriesPerformanceTest.java index d81eea997c..a2abfc28b6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/sql/LatestTimeseriesPerformanceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/sql/LatestTimeseriesPerformanceTest.java @@ -56,7 +56,8 @@ public class LatestTimeseriesPerformanceTest extends AbstractServiceTest { private static final String LONG_KEY = "longKey"; private static final String DOUBLE_KEY = "doubleKey"; private static final String BOOLEAN_KEY = "booleanKey"; - public static final int AMOUNT_OF_UNIQ_KEY = 10000; + private static final int AMOUNT_OF_UNIQ_KEY = 10000; + private static final int TIMEOUT = 100; private final Random random = new Random(); @@ -102,7 +103,7 @@ public class LatestTimeseriesPerformanceTest extends AbstractServiceTest { futures.add(save(generateDblEntry(getRandomKey()))); futures.add(save(generateBoolEntry(getRandomKey()))); } - Futures.allAsList(futures).get(60, TimeUnit.SECONDS); + Futures.allAsList(futures).get(TIMEOUT, TimeUnit.SECONDS); long endTime = System.currentTimeMillis(); long totalTime = endTime - startTime; @@ -120,7 +121,7 @@ public class LatestTimeseriesPerformanceTest extends AbstractServiceTest { futures.add(save(generateDblEntry(i))); futures.add(save(generateBoolEntry(i))); } - Futures.allAsList(futures).get(60, TimeUnit.SECONDS); + Futures.allAsList(futures).get(TIMEOUT, TimeUnit.SECONDS); } private ListenableFuture save(TsKvEntry tsKvEntry) { From 4219ed075f4c3ad56c22c49371406b8c09302e3e Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 22 Nov 2024 16:29:29 +0200 Subject: [PATCH 21/37] UI: Fixed add small widget in mobile layout dashboard --- .../src/app/modules/home/models/dashboard-component.models.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/models/dashboard-component.models.ts b/ui-ngx/src/app/modules/home/models/dashboard-component.models.ts index b4132bf882..1f8c4c91f7 100644 --- a/ui-ngx/src/app/modules/home/models/dashboard-component.models.ts +++ b/ui-ngx/src/app/modules/home/models/dashboard-component.models.ts @@ -761,7 +761,8 @@ export class DashboardWidget implements GridsterItem, IDashboardWidget { } else { res = this.sizeY; } - return Math.floor(res); + res = Math.floor(res); + return res === 0 ? 1 : res; } set rows(rows: number) { From 936ad3c2b252e3f94edcc7dc371862ba27fb6de1 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Mon, 25 Nov 2024 12:19:47 +0200 Subject: [PATCH 22/37] Refactored due to comments --- .../transport/mqtt/MqttTransportHandler.java | 34 +++++++------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 6d5c0f5d50..e2bc8cd768 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -314,26 +314,8 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } break; case SUBSCRIBE: - MqttSubscribeMessage mqttSubscribeMsg = (MqttSubscribeMessage) msg; - log.trace("[{}] Processing subscription [{}]!", sessionId, mqttSubscribeMsg.variableHeader().messageId()); - List grantedQoSList = new ArrayList<>(); - for (MqttTopicSubscription subscription : mqttSubscribeMsg.payload().topicSubscriptions()) { - String topic = subscription.topicFilter(); - MqttQoS reqQoS = subscription.qualityOfService(); - try { - if (MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC.equals(topic)) { - registerSubQoS(topic, grantedQoSList, reqQoS); - } else { - log.warn("[{}] Failed to subscribe to [{}][{}]", sessionId, topic, reqQoS); - grantedQoSList.add(ReturnCodeResolver.getSubscriptionReturnCode(deviceSessionCtx.getMqttVersion(), MqttReasonCodes.SubAck.TOPIC_FILTER_INVALID)); - break; - } - } catch (Exception e) { - log.warn("[{}] Failed to subscribe to [{}][{}]", sessionId, topic, reqQoS, e); - grantedQoSList.add(ReturnCodeResolver.getSubscriptionReturnCode(deviceSessionCtx.getMqttVersion(), MqttReasonCodes.SubAck.IMPLEMENTATION_SPECIFIC_ERROR)); - } - } - ctx.writeAndFlush(createSubAckMessage(mqttSubscribeMsg.variableHeader().messageId(), grantedQoSList)); + MqttSubscribeMessage subscribeMessage = (MqttSubscribeMessage) msg; + processSubscribe(ctx, subscribeMessage); break; case PINGREQ: ctx.writeAndFlush(new MqttMessage(new MqttFixedHeader(PINGRESP, false, AT_MOST_ONCE, false, 0))); @@ -772,7 +754,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } private void processSubscribe(ChannelHandlerContext ctx, MqttSubscribeMessage mqttMsg) { - if (!checkConnected(ctx, mqttMsg)) { + if (!checkConnected(ctx, mqttMsg) && !deviceSessionCtx.isProvisionOnly()) { ctx.writeAndFlush(createSubAckMessage(mqttMsg.variableHeader().messageId(), Collections.singletonList(MqttReasonCodes.SubAck.NOT_AUTHORIZED.byteValue() & 0xFF))); return; } @@ -782,6 +764,16 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement for (MqttTopicSubscription subscription : mqttMsg.payload().topicSubscriptions()) { String topic = subscription.topicName(); MqttQoS reqQoS = subscription.qualityOfService(); + if (deviceSessionCtx.isProvisionOnly()) { + if (MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC.equals(topic)) { + registerSubQoS(topic, grantedQoSList, reqQoS); + } else { + log.debug("[{}] Failed to subscribe to [{}][{}]", sessionId, topic, reqQoS); + grantedQoSList.add(ReturnCodeResolver.getSubscriptionReturnCode(deviceSessionCtx.getMqttVersion(), MqttReasonCodes.SubAck.TOPIC_FILTER_INVALID)); + } + activityReported = true; + continue; + } if (deviceSessionCtx.isDeviceSubscriptionAttributesTopic(topic)) { processAttributesSubscribe(grantedQoSList, topic, reqQoS, TopicType.V1); activityReported = true; From 44930bf9a1e777e088370cc38903249257a54f23 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Mon, 25 Nov 2024 13:39:13 +0200 Subject: [PATCH 23/37] Added processing for unsubscribe for provisioning only clients --- .../server/transport/mqtt/MqttTransportHandler.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index e2bc8cd768..ed721954a7 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -886,7 +886,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } private void processUnsubscribe(ChannelHandlerContext ctx, MqttUnsubscribeMessage mqttMsg) { - if (!checkConnected(ctx, mqttMsg)) { + if (!checkConnected(ctx, mqttMsg) && !deviceSessionCtx.isProvisionOnly()) { ctx.writeAndFlush(createUnSubAckMessage(mqttMsg.variableHeader().messageId(), Collections.singletonList((short) MqttReasonCodes.UnsubAck.NOT_AUTHORIZED.byteValue()))); return; @@ -900,6 +900,14 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement mqttQoSMap.remove(matcher); try { short resultValue = MqttReasonCodes.UnsubAck.SUCCESS.byteValue(); + if (deviceSessionCtx.isProvisionOnly()) { + if (!matcher.matches(MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC)) { + resultValue = MqttReasonCodes.UnsubAck.TOPIC_FILTER_INVALID.byteValue(); + } + unSubResults.add(resultValue); + activityReported = true; + continue; + } switch (topicName) { case MqttTopics.DEVICE_ATTRIBUTES_TOPIC: case MqttTopics.DEVICE_ATTRIBUTES_SHORT_TOPIC: @@ -930,7 +938,6 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement case MqttTopics.GATEWAY_ATTRIBUTES_TOPIC: case MqttTopics.GATEWAY_RPC_TOPIC: case MqttTopics.GATEWAY_ATTRIBUTES_RESPONSE_TOPIC: - case MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC: case MqttTopics.DEVICE_FIRMWARE_RESPONSES_TOPIC: case MqttTopics.DEVICE_FIRMWARE_ERROR_TOPIC: case MqttTopics.DEVICE_SOFTWARE_RESPONSES_TOPIC: From 52b2c5f9ebe5d3eeb9292dd736c8348f73ec32b4 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Mon, 25 Nov 2024 13:42:57 +0200 Subject: [PATCH 24/37] Added forgotten case --- .../server/transport/mqtt/MqttTransportHandler.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index ed721954a7..54df37c3f6 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -317,6 +317,10 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement MqttSubscribeMessage subscribeMessage = (MqttSubscribeMessage) msg; processSubscribe(ctx, subscribeMessage); break; + case UNSUBSCRIBE: + MqttUnsubscribeMessage unsubscribeMessage = (MqttUnsubscribeMessage) msg; + processUnsubscribe(ctx, unsubscribeMessage); + break; case PINGREQ: ctx.writeAndFlush(new MqttMessage(new MqttFixedHeader(PINGRESP, false, AT_MOST_ONCE, false, 0))); break; From 4baf35c8541020735b7534229638ef1a3ee5fcbc Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 25 Nov 2024 13:50:38 +0200 Subject: [PATCH 25/37] UI: Set esbuild to 0.23.0 --- ui-ngx/package.json | 3 +- ui-ngx/yarn.lock | 146 +------------------------------------------- 2 files changed, 3 insertions(+), 146 deletions(-) diff --git a/ui-ngx/package.json b/ui-ngx/package.json index f6c278b71c..0ac7b1e4e2 100644 --- a/ui-ngx/package.json +++ b/ui-ngx/package.json @@ -168,6 +168,7 @@ "ace-builds": "1.36.2", "tinymce": "6.8.5", "rollup": "4.22.4", - "@babel/core": "7.25.2" + "@babel/core": "7.25.2", + "esbuild": "0.23.0" } } diff --git a/ui-ngx/yarn.lock b/ui-ngx/yarn.lock index 32caafc402..c51b279dce 100644 --- a/ui-ngx/yarn.lock +++ b/ui-ngx/yarn.lock @@ -1498,181 +1498,91 @@ esquery "^1.6.0" jsdoc-type-pratt-parser "~4.1.0" -"@esbuild/aix-ppc64@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz#c7184a326533fcdf1b8ee0733e21c713b975575f" - integrity sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ== - "@esbuild/aix-ppc64@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.23.0.tgz#145b74d5e4a5223489cabdc238d8dad902df5259" integrity sha512-3sG8Zwa5fMcA9bgqB8AfWPQ+HFke6uD3h1s3RIwUNK8EG7a4buxvuFTs3j1IMs2NXAk9F30C/FF4vxRgQCcmoQ== -"@esbuild/android-arm64@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz#09d9b4357780da9ea3a7dfb833a1f1ff439b4052" - integrity sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A== - "@esbuild/android-arm64@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.23.0.tgz#453bbe079fc8d364d4c5545069e8260228559832" integrity sha512-EuHFUYkAVfU4qBdyivULuu03FhJO4IJN9PGuABGrFy4vUuzk91P2d+npxHcFdpUnfYKy0PuV+n6bKIpHOB3prQ== -"@esbuild/android-arm@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.21.5.tgz#9b04384fb771926dfa6d7ad04324ecb2ab9b2e28" - integrity sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg== - "@esbuild/android-arm@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.23.0.tgz#26c806853aa4a4f7e683e519cd9d68e201ebcf99" integrity sha512-+KuOHTKKyIKgEEqKbGTK8W7mPp+hKinbMBeEnNzjJGyFcWsfrXjSTNluJHCY1RqhxFurdD8uNXQDei7qDlR6+g== -"@esbuild/android-x64@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.21.5.tgz#29918ec2db754cedcb6c1b04de8cd6547af6461e" - integrity sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA== - "@esbuild/android-x64@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.23.0.tgz#1e51af9a6ac1f7143769f7ee58df5b274ed202e6" integrity sha512-WRrmKidLoKDl56LsbBMhzTTBxrsVwTKdNbKDalbEZr0tcsBgCLbEtoNthOW6PX942YiYq8HzEnb4yWQMLQuipQ== -"@esbuild/darwin-arm64@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz#e495b539660e51690f3928af50a76fb0a6ccff2a" - integrity sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ== - "@esbuild/darwin-arm64@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.23.0.tgz#d996187a606c9534173ebd78c58098a44dd7ef9e" integrity sha512-YLntie/IdS31H54Ogdn+v50NuoWF5BDkEUFpiOChVa9UnKpftgwzZRrI4J132ETIi+D8n6xh9IviFV3eXdxfow== -"@esbuild/darwin-x64@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz#c13838fa57372839abdddc91d71542ceea2e1e22" - integrity sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw== - "@esbuild/darwin-x64@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.23.0.tgz#30c8f28a7ef4e32fe46501434ebe6b0912e9e86c" integrity sha512-IMQ6eme4AfznElesHUPDZ+teuGwoRmVuuixu7sv92ZkdQcPbsNHzutd+rAfaBKo8YK3IrBEi9SLLKWJdEvJniQ== -"@esbuild/freebsd-arm64@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz#646b989aa20bf89fd071dd5dbfad69a3542e550e" - integrity sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g== - "@esbuild/freebsd-arm64@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.23.0.tgz#30f4fcec8167c08a6e8af9fc14b66152232e7fb4" integrity sha512-0muYWCng5vqaxobq6LB3YNtevDFSAZGlgtLoAc81PjUfiFz36n4KMpwhtAd4he8ToSI3TGyuhyx5xmiWNYZFyw== -"@esbuild/freebsd-x64@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz#aa615cfc80af954d3458906e38ca22c18cf5c261" - integrity sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ== - "@esbuild/freebsd-x64@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.23.0.tgz#1003a6668fe1f5d4439e6813e5b09a92981bc79d" integrity sha512-XKDVu8IsD0/q3foBzsXGt/KjD/yTKBCIwOHE1XwiXmrRwrX6Hbnd5Eqn/WvDekddK21tfszBSrE/WMaZh+1buQ== -"@esbuild/linux-arm64@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz#70ac6fa14f5cb7e1f7f887bcffb680ad09922b5b" - integrity sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q== - "@esbuild/linux-arm64@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.23.0.tgz#3b9a56abfb1410bb6c9138790f062587df3e6e3a" integrity sha512-j1t5iG8jE7BhonbsEg5d9qOYcVZv/Rv6tghaXM/Ug9xahM0nX/H2gfu6X6z11QRTMT6+aywOMA8TDkhPo8aCGw== -"@esbuild/linux-arm@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz#fc6fd11a8aca56c1f6f3894f2bea0479f8f626b9" - integrity sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA== - "@esbuild/linux-arm@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.23.0.tgz#237a8548e3da2c48cd79ae339a588f03d1889aad" integrity sha512-SEELSTEtOFu5LPykzA395Mc+54RMg1EUgXP+iw2SJ72+ooMwVsgfuwXo5Fn0wXNgWZsTVHwY2cg4Vi/bOD88qw== -"@esbuild/linux-ia32@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz#3271f53b3f93e3d093d518d1649d6d68d346ede2" - integrity sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg== - "@esbuild/linux-ia32@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.23.0.tgz#4269cd19cb2de5de03a7ccfc8855dde3d284a238" integrity sha512-P7O5Tkh2NbgIm2R6x1zGJJsnacDzTFcRWZyTTMgFdVit6E98LTxO+v8LCCLWRvPrjdzXHx9FEOA8oAZPyApWUA== -"@esbuild/linux-loong64@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz#ed62e04238c57026aea831c5a130b73c0f9f26df" - integrity sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg== - "@esbuild/linux-loong64@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.23.0.tgz#82b568f5658a52580827cc891cb69d2cb4f86280" integrity sha512-InQwepswq6urikQiIC/kkx412fqUZudBO4SYKu0N+tGhXRWUqAx+Q+341tFV6QdBifpjYgUndV1hhMq3WeJi7A== -"@esbuild/linux-mips64el@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz#e79b8eb48bf3b106fadec1ac8240fb97b4e64cbe" - integrity sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg== - "@esbuild/linux-mips64el@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.23.0.tgz#9a57386c926262ae9861c929a6023ed9d43f73e5" integrity sha512-J9rflLtqdYrxHv2FqXE2i1ELgNjT+JFURt/uDMoPQLcjWQA5wDKgQA4t/dTqGa88ZVECKaD0TctwsUfHbVoi4w== -"@esbuild/linux-ppc64@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz#5f2203860a143b9919d383ef7573521fb154c3e4" - integrity sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w== - "@esbuild/linux-ppc64@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.23.0.tgz#f3a79fd636ba0c82285d227eb20ed8e31b4444f6" integrity sha512-cShCXtEOVc5GxU0fM+dsFD10qZ5UpcQ8AM22bYj0u/yaAykWnqXJDpd77ublcX6vdDsWLuweeuSNZk4yUxZwtw== -"@esbuild/linux-riscv64@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz#07bcafd99322d5af62f618cb9e6a9b7f4bb825dc" - integrity sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA== - "@esbuild/linux-riscv64@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.23.0.tgz#f9d2ef8356ce6ce140f76029680558126b74c780" integrity sha512-HEtaN7Y5UB4tZPeQmgz/UhzoEyYftbMXrBCUjINGjh3uil+rB/QzzpMshz3cNUxqXN7Vr93zzVtpIDL99t9aRw== -"@esbuild/linux-s390x@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz#b7ccf686751d6a3e44b8627ababc8be3ef62d8de" - integrity sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A== - "@esbuild/linux-s390x@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.23.0.tgz#45390f12e802201f38a0229e216a6aed4351dfe8" integrity sha512-WDi3+NVAuyjg/Wxi+o5KPqRbZY0QhI9TjrEEm+8dmpY9Xir8+HE/HNx2JoLckhKbFopW0RdO2D72w8trZOV+Wg== -"@esbuild/linux-x64@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz#6d8f0c768e070e64309af8004bb94e68ab2bb3b0" - integrity sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ== - "@esbuild/linux-x64@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.23.0.tgz#c8409761996e3f6db29abcf9b05bee8d7d80e910" integrity sha512-a3pMQhUEJkITgAw6e0bWA+F+vFtCciMjW/LPtoj99MhVt+Mfb6bbL9hu2wmTZgNd994qTAEw+U/r6k3qHWWaOQ== -"@esbuild/netbsd-x64@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz#bbe430f60d378ecb88decb219c602667387a6047" - integrity sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg== - "@esbuild/netbsd-x64@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.23.0.tgz#ba70db0114380d5f6cfb9003f1d378ce989cd65c" @@ -1683,51 +1593,26 @@ resolved "https://registry.yarnpkg.com/@esbuild/openbsd-arm64/-/openbsd-arm64-0.23.0.tgz#72fc55f0b189f7a882e3cf23f332370d69dfd5db" integrity sha512-suXjq53gERueVWu0OKxzWqk7NxiUWSUlrxoZK7usiF50C6ipColGR5qie2496iKGYNLhDZkPxBI3erbnYkU0rQ== -"@esbuild/openbsd-x64@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz#99d1cf2937279560d2104821f5ccce220cb2af70" - integrity sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow== - "@esbuild/openbsd-x64@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.23.0.tgz#b6ae7a0911c18fe30da3db1d6d17a497a550e5d8" integrity sha512-6p3nHpby0DM/v15IFKMjAaayFhqnXV52aEmv1whZHX56pdkK+MEaLoQWj+H42ssFarP1PcomVhbsR4pkz09qBg== -"@esbuild/sunos-x64@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz#08741512c10d529566baba837b4fe052c8f3487b" - integrity sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg== - "@esbuild/sunos-x64@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.23.0.tgz#58f0d5e55b9b21a086bfafaa29f62a3eb3470ad8" integrity sha512-BFelBGfrBwk6LVrmFzCq1u1dZbG4zy/Kp93w2+y83Q5UGYF1d8sCzeLI9NXjKyujjBBniQa8R8PzLFAUrSM9OA== -"@esbuild/win32-arm64@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz#675b7385398411240735016144ab2e99a60fc75d" - integrity sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A== - "@esbuild/win32-arm64@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.23.0.tgz#b858b2432edfad62e945d5c7c9e5ddd0f528ca6d" integrity sha512-lY6AC8p4Cnb7xYHuIxQ6iYPe6MfO2CC43XXKo9nBXDb35krYt7KGhQnOkRGar5psxYkircpCqfbNDB4uJbS2jQ== -"@esbuild/win32-ia32@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz#1bfc3ce98aa6ca9a0969e4d2af72144c59c1193b" - integrity sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA== - "@esbuild/win32-ia32@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.23.0.tgz#167ef6ca22a476c6c0c014a58b4f43ae4b80dec7" integrity sha512-7L1bHlOTcO4ByvI7OXVI5pNN6HSu6pUQq9yodga8izeuB1KcT2UkHaH6118QJwopExPn0rMHIseCTx1CRo/uNA== -"@esbuild/win32-x64@0.21.5": - version "0.21.5" - resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz#acad351d582d157bb145535db2a6ff53dd514b5c" - integrity sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw== - "@esbuild/win32-x64@0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.23.0.tgz#db44a6a08520b5f25bbe409f34a59f2d4bcc7ced" @@ -5447,7 +5332,7 @@ esbuild-wasm@0.23.0: resolved "https://registry.yarnpkg.com/esbuild-wasm/-/esbuild-wasm-0.23.0.tgz#7b09c7bc669b702b440aeb3d5c9210f96766e1b2" integrity sha512-6jP8UmWy6R6TUUV8bMuC3ZyZ6lZKI56x0tkxyCIqWwRRJ/DgeQKneh/Oid5EoGoPFLrGNkz47ZEtWAYuiY/u9g== -esbuild@0.23.0: +esbuild@0.23.0, esbuild@^0.21.3: version "0.23.0" resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.23.0.tgz#de06002d48424d9fdb7eb52dbe8e95927f852599" integrity sha512-1lvV17H2bMYda/WaFb2jLPeHU3zml2k4/yagNMG8Q/YtfMjCwEUZa2eXXMgZTVSL5q1n4H7sQ0X6CdJDqqeCFA== @@ -5477,35 +5362,6 @@ esbuild@0.23.0: "@esbuild/win32-ia32" "0.23.0" "@esbuild/win32-x64" "0.23.0" -esbuild@^0.21.3: - version "0.21.5" - resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.21.5.tgz#9ca301b120922959b766360d8ac830da0d02997d" - integrity sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw== - optionalDependencies: - "@esbuild/aix-ppc64" "0.21.5" - "@esbuild/android-arm" "0.21.5" - "@esbuild/android-arm64" "0.21.5" - "@esbuild/android-x64" "0.21.5" - "@esbuild/darwin-arm64" "0.21.5" - "@esbuild/darwin-x64" "0.21.5" - "@esbuild/freebsd-arm64" "0.21.5" - "@esbuild/freebsd-x64" "0.21.5" - "@esbuild/linux-arm" "0.21.5" - "@esbuild/linux-arm64" "0.21.5" - "@esbuild/linux-ia32" "0.21.5" - "@esbuild/linux-loong64" "0.21.5" - "@esbuild/linux-mips64el" "0.21.5" - "@esbuild/linux-ppc64" "0.21.5" - "@esbuild/linux-riscv64" "0.21.5" - "@esbuild/linux-s390x" "0.21.5" - "@esbuild/linux-x64" "0.21.5" - "@esbuild/netbsd-x64" "0.21.5" - "@esbuild/openbsd-x64" "0.21.5" - "@esbuild/sunos-x64" "0.21.5" - "@esbuild/win32-arm64" "0.21.5" - "@esbuild/win32-ia32" "0.21.5" - "@esbuild/win32-x64" "0.21.5" - escalade@^3.1.1, escalade@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.2.0.tgz#011a3f69856ba189dffa7dc8fcce99d2a87903e5" From 5916f3114317874b066731b48c93d181703566e7 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Mon, 25 Nov 2024 13:56:30 +0200 Subject: [PATCH 26/37] UI: Refactoring calc row in DashboardWidget --- .../src/app/modules/home/models/dashboard-component.models.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/models/dashboard-component.models.ts b/ui-ngx/src/app/modules/home/models/dashboard-component.models.ts index 1f8c4c91f7..6658f0b8a4 100644 --- a/ui-ngx/src/app/modules/home/models/dashboard-component.models.ts +++ b/ui-ngx/src/app/modules/home/models/dashboard-component.models.ts @@ -761,8 +761,7 @@ export class DashboardWidget implements GridsterItem, IDashboardWidget { } else { res = this.sizeY; } - res = Math.floor(res); - return res === 0 ? 1 : res; + return Math.max(Math.floor(res), 1); } set rows(rows: number) { From c8d627371b0725a95d7a3acdc882c4701e28593f Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 25 Nov 2024 18:48:01 +0200 Subject: [PATCH 27/37] fixed oauth2 authorization for apps of version 1.3.0 and less --- ...tomOAuth2AuthorizationRequestResolver.java | 36 +++++++++++++++---- .../dao/oauth2/OAuth2ClientService.java | 2 +- .../server/dao/oauth2/OAuth2ClientDao.java | 2 +- .../dao/oauth2/OAuth2ClientServiceImpl.java | 10 ++++-- .../dao/sql/oauth2/JpaOAuth2ClientDao.java | 4 +-- .../sql/oauth2/OAuth2ClientRepository.java | 5 +-- 6 files changed, 43 insertions(+), 16 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java b/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java index 542a7c71cb..f8935f00d6 100644 --- a/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java +++ b/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java @@ -38,6 +38,7 @@ import org.springframework.web.util.UriComponents; import org.springframework.web.util.UriComponentsBuilder; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.OAuth2ClientId; +import org.thingsboard.server.common.data.oauth2.PlatformType; import org.thingsboard.server.dao.oauth2.OAuth2Configuration; import org.thingsboard.server.dao.oauth2.OAuth2ClientService; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -85,8 +86,9 @@ public class CustomOAuth2AuthorizationRequestResolver implements OAuth2Authoriza String registrationId = this.resolveRegistrationId(request); String redirectUriAction = getAction(request, "login"); String appPackage = getAppPackage(request); + String platform = getPlatform(request); String appToken = getAppToken(request); - return resolve(request, registrationId, redirectUriAction, appPackage, appToken); + return resolve(request, registrationId, redirectUriAction, appPackage, platform, appToken); } @Override @@ -96,8 +98,9 @@ public class CustomOAuth2AuthorizationRequestResolver implements OAuth2Authoriza } String redirectUriAction = getAction(request, "authorize"); String appPackage = getAppPackage(request); + String platform = getPlatform(request); String appToken = getAppToken(request); - return resolve(request, registrationId, redirectUriAction, appPackage, appToken); + return resolve(request, registrationId, redirectUriAction, appPackage, platform, appToken); } private String getAction(HttpServletRequest request, String defaultAction) { @@ -112,11 +115,15 @@ public class CustomOAuth2AuthorizationRequestResolver implements OAuth2Authoriza return request.getParameter("pkg"); } + private String getPlatform(HttpServletRequest request) { + return request.getParameter("platform"); + } + private String getAppToken(HttpServletRequest request) { return request.getParameter("appToken"); } - private OAuth2AuthorizationRequest resolve(HttpServletRequest request, String oauth2ClientId, String redirectUriAction, String appPackage, String appToken) { + private OAuth2AuthorizationRequest resolve(HttpServletRequest request, String oauth2ClientId, String redirectUriAction, String appPackage, String platform, String appToken) { if (oauth2ClientId == null) { return null; } @@ -132,11 +139,18 @@ public class CustomOAuth2AuthorizationRequestResolver implements OAuth2Authoriza if (StringUtils.isEmpty(appToken)) { throw new IllegalArgumentException("Invalid application token."); } else { - String appSecret = this.oAuth2ClientService.findAppSecret(new OAuth2ClientId(UUID.fromString(oauth2ClientId)), appPackage); - if (StringUtils.isEmpty(appSecret)) { - throw new IllegalArgumentException("Invalid package: " + appPackage + ". No application secret found for Client Registration with given application package."); + String callbackUrlScheme; + if (platform != null) { + callbackUrlScheme = validateMobileAppToken(oauth2ClientId, appPackage, PlatformType.valueOf(platform), appToken); + } else { + // for backward compatibility with mobile apps of version 1.3.0 and less try to validate token with android and then ios app secret + try { + callbackUrlScheme = validateMobileAppToken(oauth2ClientId, appPackage, PlatformType.ANDROID, appToken); + } catch (IllegalArgumentException e) { + log.warn("Failed attempt to validate android application token", e); + callbackUrlScheme = validateMobileAppToken(oauth2ClientId, appPackage, PlatformType.IOS, appToken); + } } - String callbackUrlScheme = this.oAuth2AppTokenFactory.validateTokenAndGetCallbackUrlScheme(appPackage, appToken, appSecret); attributes.put(TbOAuth2ParameterNames.CALLBACK_URL_SCHEME, callbackUrlScheme); } } @@ -174,6 +188,14 @@ public class CustomOAuth2AuthorizationRequestResolver implements OAuth2Authoriza .build(); } + private String validateMobileAppToken(String oauth2ClientId, String appPackage, PlatformType platformType, String appToken) { + String appSecret = this.oAuth2ClientService.findAppSecret(new OAuth2ClientId(UUID.fromString(oauth2ClientId)), appPackage, platformType); + if (StringUtils.isEmpty(appSecret)) { + throw new IllegalArgumentException("Invalid package: " + appPackage + ". No application secret found for Client Registration with given application package."); + } + return this.oAuth2AppTokenFactory.validateTokenAndGetCallbackUrlScheme(appPackage, appToken, appSecret); + } + private String resolveRegistrationId(HttpServletRequest request) { if (this.authorizationRequestMatcher.matches(request)) { return this.authorizationRequestMatcher diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientService.java index 317221fe0a..f293462cc6 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientService.java @@ -40,7 +40,7 @@ public interface OAuth2ClientService extends EntityDaoService { OAuth2Client findOAuth2ClientById(TenantId tenantId, OAuth2ClientId providerId); - String findAppSecret(OAuth2ClientId oAuth2ClientId, String pkgName); + String findAppSecret(OAuth2ClientId oAuth2ClientId, String pkgName, PlatformType platformType); void deleteOAuth2ClientById(TenantId tenantId, OAuth2ClientId oAuth2ClientId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientDao.java b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientDao.java index 01d56feb20..16dda80840 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientDao.java @@ -38,7 +38,7 @@ public interface OAuth2ClientDao extends Dao { List findByMobileAppBundleId(UUID mobileAppBundleId); - String findAppSecret(UUID id, String pkgName); + String findAppSecret(UUID id, String pkgName, PlatformType platformType); void deleteByTenantId(UUID tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientServiceImpl.java index 880329335c..683c323ab5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientServiceImpl.java @@ -34,6 +34,7 @@ import org.thingsboard.server.dao.entity.AbstractEntityService; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.service.DataValidator; +import org.thingsboard.server.dao.service.Validator; import java.util.Comparator; import java.util.List; @@ -45,6 +46,8 @@ import java.util.stream.Collectors; @Service("OAuth2ClientService") public class OAuth2ClientServiceImpl extends AbstractEntityService implements OAuth2ClientService { + private static final String PLATFORM_TYPE_IS_REQUIRED = "Platform type is required if package name is specified"; + @Autowired private OAuth2ClientDao oauth2ClientDao; @Autowired @@ -90,9 +93,10 @@ public class OAuth2ClientServiceImpl extends AbstractEntityService implements OA } @Override - public String findAppSecret(OAuth2ClientId oAuth2ClientId, String pkgName) { - log.trace("Executing findAppSecret oAuth2ClientId = [{}] pkgName = [{}]", oAuth2ClientId, pkgName); - return oauth2ClientDao.findAppSecret(oAuth2ClientId.getId(), pkgName); + public String findAppSecret(OAuth2ClientId oAuth2ClientId, String pkgName, PlatformType platformType) { + log.trace("Executing findAppSecret oAuth2ClientId = [{}] pkgName = [{}], platform [{}]", oAuth2ClientId, pkgName, platformType); + Validator.checkNotNull(platformType, PLATFORM_TYPE_IS_REQUIRED); + return oauth2ClientDao.findAppSecret(oAuth2ClientId.getId(), pkgName, platformType); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/oauth2/JpaOAuth2ClientDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/oauth2/JpaOAuth2ClientDao.java index d694ca0f93..0db0faed69 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/oauth2/JpaOAuth2ClientDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/oauth2/JpaOAuth2ClientDao.java @@ -80,8 +80,8 @@ public class JpaOAuth2ClientDao extends JpaAbstractDao Date: Wed, 27 Nov 2024 11:26:56 +0200 Subject: [PATCH 28/37] updated log level and message, deleted redundant null-check --- .../config/CustomOAuth2AuthorizationRequestResolver.java | 3 ++- .../thingsboard/server/dao/sql/oauth2/JpaOAuth2ClientDao.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java b/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java index f8935f00d6..c0dfaa599a 100644 --- a/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java +++ b/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java @@ -147,7 +147,8 @@ public class CustomOAuth2AuthorizationRequestResolver implements OAuth2Authoriza try { callbackUrlScheme = validateMobileAppToken(oauth2ClientId, appPackage, PlatformType.ANDROID, appToken); } catch (IllegalArgumentException e) { - log.warn("Failed attempt to validate android application token", e); + log.debug("Failed attempt to validate android application token, oauth client id: [{}], package name: [{}], appToken [{}] ", + oauth2ClientId, appPackage, appToken, e); callbackUrlScheme = validateMobileAppToken(oauth2ClientId, appPackage, PlatformType.IOS, appToken); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/oauth2/JpaOAuth2ClientDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/oauth2/JpaOAuth2ClientDao.java index 0db0faed69..6febd651bb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/oauth2/JpaOAuth2ClientDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/oauth2/JpaOAuth2ClientDao.java @@ -81,7 +81,7 @@ public class JpaOAuth2ClientDao extends JpaAbstractDao Date: Wed, 27 Nov 2024 12:51:47 +0200 Subject: [PATCH 29/37] allow mobile deletion if used by mobile bundle --- .../main/data/upgrade/3.8.1/schema_update.sql | 4 ++-- .../controller/MobileAppController.java | 23 ++++++++----------- .../common/data/mobile/LoginMobileInfo.java | 5 +++- .../common/data/mobile/UserMobileInfo.java | 8 ++++++- .../main/resources/sql/schema-entities.sql | 4 ++-- 5 files changed, 25 insertions(+), 19 deletions(-) diff --git a/application/src/main/data/upgrade/3.8.1/schema_update.sql b/application/src/main/data/upgrade/3.8.1/schema_update.sql index e4faca82a5..3aa0f08edf 100644 --- a/application/src/main/data/upgrade/3.8.1/schema_update.sql +++ b/application/src/main/data/upgrade/3.8.1/schema_update.sql @@ -38,8 +38,8 @@ CREATE TABLE IF NOT EXISTS mobile_app_bundle ( ios_app_id uuid UNIQUE, layout_config varchar(16384), oauth2_enabled boolean, - CONSTRAINT fk_android_app_id FOREIGN KEY (android_app_id) REFERENCES mobile_app(id), - CONSTRAINT fk_ios_app_id FOREIGN KEY (ios_app_id) REFERENCES mobile_app(id) + CONSTRAINT fk_android_app_id FOREIGN KEY (android_app_id) REFERENCES mobile_app(id) ON DELETE SET NULL, + CONSTRAINT fk_ios_app_id FOREIGN KEY (ios_app_id) REFERENCES mobile_app(id) ON DELETE SET NULL ); CREATE INDEX IF NOT EXISTS mobile_app_bundle_tenant_id ON mobile_app_bundle(tenant_id); diff --git a/application/src/main/java/org/thingsboard/server/controller/MobileAppController.java b/application/src/main/java/org/thingsboard/server/controller/MobileAppController.java index da206193fb..3626ff4c3e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/MobileAppController.java +++ b/application/src/main/java/org/thingsboard/server/controller/MobileAppController.java @@ -41,6 +41,7 @@ import org.thingsboard.server.common.data.mobile.LoginMobileInfo; import org.thingsboard.server.common.data.mobile.UserMobileInfo; import org.thingsboard.server.common.data.mobile.app.MobileApp; import org.thingsboard.server.common.data.mobile.app.MobileAppVersionInfo; +import org.thingsboard.server.common.data.mobile.app.StoreInfo; import org.thingsboard.server.common.data.mobile.bundle.MobileAppBundle; import org.thingsboard.server.common.data.mobile.layout.MobilePage; import org.thingsboard.server.common.data.oauth2.OAuth2ClientLoginInfo; @@ -55,6 +56,7 @@ import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; @@ -80,10 +82,12 @@ public class MobileAppController extends BaseController { public LoginMobileInfo getLoginMobileInfo(@Parameter(description = "Mobile application package name") @RequestParam String pkgName, @Parameter(description = "Platform type", schema = @Schema(allowableValues = {"ANDROID", "IOS"})) - @RequestParam PlatformType platform) { + @RequestParam PlatformType platform) throws ThingsboardException { List oauth2Clients = oAuth2ClientService.findOAuth2ClientLoginInfosByMobilePkgNameAndPlatformType(pkgName, platform); - MobileApp mobileApp = mobileAppService.findMobileAppByPkgNameAndPlatformType(pkgName, platform); - return new LoginMobileInfo(oauth2Clients, mobileApp != null ? mobileApp.getVersionInfo() : null); + MobileApp mobileApp = checkNotNull(mobileAppService.findMobileAppByPkgNameAndPlatformType(pkgName, platform)); + StoreInfo storeInfo = Optional.ofNullable(mobileApp).map(MobileApp::getStoreInfo).orElse(null); + MobileAppVersionInfo versionInfo = Optional.ofNullable(mobileApp).map(MobileApp::getVersionInfo).orElse(null); + return new LoginMobileInfo(oauth2Clients, storeInfo, versionInfo); } @ApiOperation(value = "Get user mobile app basic info (getUserMobileInfo)", notes = AVAILABLE_FOR_ANY_AUTHORIZED_USER) @@ -97,17 +101,10 @@ public class MobileAppController extends BaseController { User user = userService.findUserById(securityUser.getTenantId(), securityUser.getId()); HomeDashboardInfo homeDashboardInfo = securityUser.isSystemAdmin() ? null : getHomeDashboardInfo(securityUser, user.getAdditionalInfo()); MobileAppBundle mobileAppBundle = mobileAppBundleService.findMobileAppBundleByPkgNameAndPlatform(securityUser.getTenantId(), pkgName, platform); - return new UserMobileInfo(user, homeDashboardInfo, getVisiblePages(mobileAppBundle)); - } - - @ApiOperation(value = "Get mobile app version info (getMobileVersionInfo)") - @GetMapping(value = "/mobile/versionInfo") - public MobileAppVersionInfo getMobileVersionInfo(@Parameter(description = "Mobile application package name") - @RequestParam String pkgName, - @Parameter(description = "Platform type", schema = @Schema(allowableValues = {"ANDROID", "IOS"})) - @RequestParam PlatformType platform) { MobileApp mobileApp = mobileAppService.findMobileAppByPkgNameAndPlatformType(pkgName, platform); - return mobileApp != null ? mobileApp.getVersionInfo() : null; + StoreInfo storeInfo = Optional.ofNullable(mobileApp).map(MobileApp::getStoreInfo).orElse(null); + MobileAppVersionInfo versionInfo = Optional.ofNullable(mobileApp).map(MobileApp::getVersionInfo).orElse(null); + return new UserMobileInfo(user, storeInfo, versionInfo, homeDashboardInfo, getVisiblePages(mobileAppBundle)); } @ApiOperation(value = "Save Or update Mobile app (saveMobileApp)", diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/mobile/LoginMobileInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/mobile/LoginMobileInfo.java index 059dc17295..a28302bd58 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/mobile/LoginMobileInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/mobile/LoginMobileInfo.java @@ -16,9 +16,12 @@ package org.thingsboard.server.common.data.mobile; import org.thingsboard.server.common.data.mobile.app.MobileAppVersionInfo; +import org.thingsboard.server.common.data.mobile.app.StoreInfo; import org.thingsboard.server.common.data.oauth2.OAuth2ClientLoginInfo; import java.util.List; -public record LoginMobileInfo(List oAuth2ClientLoginInfos, MobileAppVersionInfo versionInfo) { +public record LoginMobileInfo(List oAuth2ClientLoginInfos, + StoreInfo storeInfo, + MobileAppVersionInfo versionInfo) { } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/mobile/UserMobileInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/mobile/UserMobileInfo.java index f7a4033b16..1d20f17bc1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/mobile/UserMobileInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/mobile/UserMobileInfo.java @@ -18,7 +18,13 @@ package org.thingsboard.server.common.data.mobile; import com.fasterxml.jackson.databind.JsonNode; import org.thingsboard.server.common.data.HomeDashboardInfo; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.mobile.app.MobileAppVersionInfo; +import org.thingsboard.server.common.data.mobile.app.StoreInfo; -public record UserMobileInfo(User user, HomeDashboardInfo homeDashboardInfo, JsonNode pages) { +public record UserMobileInfo(User user, + StoreInfo storeInfo, + MobileAppVersionInfo versionInfo, + HomeDashboardInfo homeDashboardInfo, + JsonNode pages) { } diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index edcb8645dd..cccaf9062d 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -641,8 +641,8 @@ CREATE TABLE IF NOT EXISTS mobile_app_bundle ( ios_app_id uuid UNIQUE, layout_config varchar(16384), oauth2_enabled boolean, - CONSTRAINT fk_android_app_id FOREIGN KEY (android_app_id) REFERENCES mobile_app(id), - CONSTRAINT fk_ios_app_id FOREIGN KEY (ios_app_id) REFERENCES mobile_app(id) + CONSTRAINT fk_android_app_id FOREIGN KEY (android_app_id) REFERENCES mobile_app(id) ON DELETE SET NULL, + CONSTRAINT fk_ios_app_id FOREIGN KEY (ios_app_id) REFERENCES mobile_app(id) ON DELETE SET NULL ); CREATE TABLE IF NOT EXISTS domain_oauth2_client ( From f09f2c7950ebb443b9911f2892688828f0daefd6 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 27 Nov 2024 13:21:40 +0200 Subject: [PATCH 30/37] deleted redundant constraint checks --- .../server/dao/mobile/MobileAppServiceImpl.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/mobile/MobileAppServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/mobile/MobileAppServiceImpl.java index efd152f944..d3434e1016 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/mobile/MobileAppServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/mobile/MobileAppServiceImpl.java @@ -43,7 +43,6 @@ import java.util.Optional; public class MobileAppServiceImpl extends AbstractEntityService implements MobileAppService { private static final String PLATFORM_TYPE_IS_REQUIRED = "Platform type is required if package name is specified"; - private static final String MOBILE_APP_BUNDLE_CONSTRAINT = "The mobile app referenced by the mobile bundle cannot be deleted!"; @Autowired private MobileAppDao mobileAppDao; @@ -68,15 +67,8 @@ public class MobileAppServiceImpl extends AbstractEntityService implements Mobil @Override public void deleteMobileAppById(TenantId tenantId, MobileAppId mobileAppId) { log.trace("Executing deleteMobileAppById [{}]", mobileAppId.getId()); - try { - mobileAppDao.removeById(tenantId, mobileAppId.getId()); - eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(mobileAppId).build()); - } catch (Exception e) { - checkConstraintViolation(e, - Map.of("fk_android_app_id", MOBILE_APP_BUNDLE_CONSTRAINT, - "fk_ios_app_id", MOBILE_APP_BUNDLE_CONSTRAINT)); - throw e; - } + mobileAppDao.removeById(tenantId, mobileAppId.getId()); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(mobileAppId).build()); } @Override From 65f94878d24f17842602a98b27d247e7aad75759 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 27 Nov 2024 13:36:48 +0200 Subject: [PATCH 31/37] removed using TTL from tenant profile --- .../TbSaveToCustomCassandraTableNode.java | 24 +++------- ...CustomCassandraTableNodeConfiguration.java | 4 +- .../engine/telemetry/TbMsgTimeseriesNode.java | 2 +- .../TbSaveToCustomCassandraTableNodeTest.java | 45 ++++--------------- 4 files changed, 17 insertions(+), 58 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index ddc4357674..7d4cd64179 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -38,10 +38,8 @@ import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; -import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.rule.RuleChainType; -import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.dao.cassandra.CassandraCluster; @@ -54,7 +52,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import static org.thingsboard.common.util.DonAsynchron.withCallback; @@ -88,7 +85,6 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { private PreparedStatement saveStmt; private ExecutorService readResultsProcessingExecutor; private Map fieldsMap; - private Integer ttl; @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { @@ -100,20 +96,10 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { if (!isTableExists()) { throw new TbNodeException("Table '" + TABLE_PREFIX + config.getTableName() + "' does not exist in Cassandra cluster."); } - ctx.addTenantProfileListener(this::onTenantProfileUpdate); - onTenantProfileUpdate(ctx.getTenantProfile()); startExecutor(); saveStmt = getSaveStmt(); } - private void onTenantProfileUpdate(TenantProfile tenantProfile) { - DefaultTenantProfileConfiguration configuration = (DefaultTenantProfileConfiguration) tenantProfile.getProfileData().getConfiguration(); - ttl = config.getDefaultTTL(); - if (ttl != null && ttl == 0) { - ttl = (int) TimeUnit.DAYS.toSeconds(configuration.getDefaultStorageTtlDays()); - } - } - @Override public void onMsg(TbContext ctx, TbMsg msg) { withCallback(save(msg, ctx), aVoid -> ctx.tellSuccess(msg), e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); @@ -189,7 +175,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { query.append("?, "); } } - if (ttl != null && ttl > 0) { + if (config.getDefaultTtL() > 0) { query.append(" USING TTL ?"); } return query.toString(); @@ -233,8 +219,8 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } i.getAndIncrement(); }); - if (ttl != null && ttl > 0) { - stmtBuilder.setInt(i.get(), ttl); + if (config.getDefaultTtL() > 0) { + stmtBuilder.setInt(i.get(), config.getDefaultTtL()); } return getFuture(executeAsyncWrite(ctx, stmtBuilder.build()), rs -> null); } @@ -281,9 +267,9 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { boolean hasChanges = false; switch (fromVersion) { case 0: - if (!oldConfiguration.has("defaultTTL")) { + if (!oldConfiguration.has("defaultTtL")) { hasChanges = true; - ((ObjectNode) oldConfiguration).putNull("defaultTTL"); + ((ObjectNode) oldConfiguration).put("defaultTtL", 0); } break; default: diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java index 3efae275da..e391e178f5 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java @@ -27,13 +27,13 @@ public class TbSaveToCustomCassandraTableNodeConfiguration implements NodeConfig private String tableName; private Map fieldsMapping; - private Integer defaultTTL; + private int defaultTtL; @Override public TbSaveToCustomCassandraTableNodeConfiguration defaultConfiguration() { TbSaveToCustomCassandraTableNodeConfiguration configuration = new TbSaveToCustomCassandraTableNodeConfiguration(); - configuration.setDefaultTTL(0); + configuration.setDefaultTtL(0); configuration.setTableName(""); Map map = new HashMap<>(); map.put("", ""); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index c99da715d9..c90e8e2375 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -75,7 +75,7 @@ public class TbMsgTimeseriesNode implements TbNode { onTenantProfileUpdate(ctx.getTenantProfile()); } - private void onTenantProfileUpdate(TenantProfile tenantProfile) { + void onTenantProfileUpdate(TenantProfile tenantProfile) { DefaultTenantProfileConfiguration configuration = (DefaultTenantProfileConfiguration) tenantProfile.getProfileData().getConfiguration(); tenantProfileDefaultStorageTtl = TimeUnit.DAYS.toSeconds(configuration.getDefaultStorageTtlDays()); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index f3e7d0d004..2ef5d4173c 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -46,13 +46,9 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; -import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.msg.TbMsgType; -import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; -import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.cassandra.CassandraCluster; @@ -91,7 +87,6 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("ac4ca02e-2ae6-404a-8f7e-c4ae31c56aa7")); private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("64ad971e-9cfa-49e4-9f59-faa1a2350c6e")); - private final TenantProfileId TENANT_PROFILE_ID = new TenantProfileId(UUID.fromString("239af5ad-53c1-4179-802f-3821f92ed338")); private final ListeningExecutor dbCallbackExecutor = new TestDbCallbackExecutor(); @@ -140,7 +135,7 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad public void verifyDefaultConfig() { assertThat(config.getTableName()).isEqualTo(""); assertThat(config.getFieldsMapping()).isEqualTo(Map.of("", "")); - assertThat(config.getDefaultTTL()).isEqualTo(0); + assertThat(config.getDefaultTtL()).isEqualTo(0); } @Test @@ -175,7 +170,6 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); mockCassandraCluster(); - given(ctxMock.getTenantProfile()).willReturn(getTenantProfileWithTtl(5)); assertThatThrownBy(() -> node.init(ctxMock, configuration)) .isInstanceOf(TbNodeException.class) @@ -241,16 +235,14 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad @ParameterizedTest @MethodSource - public void givenTtl_whenOnMsg_thenVerifyStatement(Integer ttlFromConfig, - int ttlFromTenantProfileInDays, + public void givenTtl_whenOnMsg_thenVerifyStatement(int ttlFromConfig, String expectedQuery, Consumer verifyBuilder) throws TbNodeException { config.setTableName("readings"); config.setFieldsMapping(Map.of("$entityId", "entityIdTableColumn")); - config.setDefaultTTL(ttlFromConfig); + config.setDefaultTtL(ttlFromConfig); mockOnInit(); - given(ctxMock.getTenantProfile()).willReturn(getTenantProfileWithTtl(ttlFromTenantProfileInDays)); willAnswer(invocation -> boundStatementBuilderMock).given(node).getStmtBuilder(); given(boundStatementBuilderMock.setUuid(anyInt(), any(UUID.class))).willReturn(boundStatementBuilderMock); given(boundStatementBuilderMock.build()).willReturn(boundStatementMock); @@ -266,28 +258,20 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad private static Stream givenTtl_whenOnMsg_thenVerifyStatement() { return Stream.of( - Arguments.of(0, 0, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?)", + Arguments.of(0, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?)", (Consumer) builder -> { then(builder).should(never()).setInt(anyInt(), anyInt()); }), - Arguments.of(0, 5, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?) USING TTL ?", - (Consumer) builder -> { - then(builder).should().setInt(1, 432000); - }), - Arguments.of(20, 1, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?) USING TTL ?", + Arguments.of(20, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?) USING TTL ?", (Consumer) builder -> { then(builder).should().setInt(1, 20); - }), - Arguments.of(null, 2, "INSERT INTO cs_tb_readings(entityIdTableColumn) VALUES(?)", - (Consumer) builder -> { - then(builder).should(never()).setInt(anyInt(), anyInt()); }) ); } @Test public void givenValidMsgStructure_whenOnMsg_thenVerifyMatchOfValuesInsertionOrderIntoStatementAndSaveToCustomCassandraTable() throws TbNodeException { - config.setDefaultTTL(25); + config.setDefaultTtL(25); config.setTableName("readings"); Map mappings = Map.of( "$entityId", "entityIdTableColumn", @@ -341,20 +325,19 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad Arguments.of(0, "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"}}", true, - "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTTL\":null}" + "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTtL\":0}" ), // default config for version 1 with upgrade from version 1 Arguments.of(1, - "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTTL\":0}", + "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTtL\":0}", false, - "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTTL\":0}" + "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTtL\":0}" ) ); } private void mockOnInit() { mockCassandraCluster(); - given(ctxMock.getTenantProfile()).willReturn(getTenantProfileWithTtl(5)); given(cassandraClusterMock.getDefaultWriteConsistencyLevel()).willReturn(DefaultConsistencyLevel.ONE); given(sessionMock.prepare(anyString())).willReturn(preparedStatementMock); } @@ -410,14 +393,4 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad then(boundStatementBuilderMock).should().setInt(values.size(), 25); } - private TenantProfile getTenantProfileWithTtl(int ttlInDays) { - var tenantProfile = new TenantProfile(TENANT_PROFILE_ID); - var tenantProfileData = new TenantProfileData(); - var tenantProfileConfiguration = new DefaultTenantProfileConfiguration(); - tenantProfileConfiguration.setDefaultStorageTtlDays(ttlInDays); - tenantProfileData.setConfiguration(tenantProfileConfiguration); - tenantProfile.setProfileData(tenantProfileData); - return tenantProfile; - } - } From 82324ff3c69d3ee667bc965ec9f52c49831c123b Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 27 Nov 2024 13:51:28 +0200 Subject: [PATCH 32/37] fixed jpa method --- .../thingsboard/server/dao/sql/oauth2/JpaOAuth2ClientDao.java | 2 +- .../server/dao/sql/oauth2/OAuth2ClientRepository.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/oauth2/JpaOAuth2ClientDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/oauth2/JpaOAuth2ClientDao.java index 6febd651bb..4310e0b705 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/oauth2/JpaOAuth2ClientDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/oauth2/JpaOAuth2ClientDao.java @@ -81,7 +81,7 @@ public class JpaOAuth2ClientDao extends JpaAbstractDao Date: Wed, 27 Nov 2024 13:57:10 +0200 Subject: [PATCH 33/37] renamed property --- .../action/TbSaveToCustomCassandraTableNode.java | 10 +++++----- ...bSaveToCustomCassandraTableNodeConfiguration.java | 4 ++-- .../action/TbSaveToCustomCassandraTableNodeTest.java | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index 7d4cd64179..a0dbc5da97 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -175,7 +175,7 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { query.append("?, "); } } - if (config.getDefaultTtL() > 0) { + if (config.getDefaultTtl() > 0) { query.append(" USING TTL ?"); } return query.toString(); @@ -219,8 +219,8 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } i.getAndIncrement(); }); - if (config.getDefaultTtL() > 0) { - stmtBuilder.setInt(i.get(), config.getDefaultTtL()); + if (config.getDefaultTtl() > 0) { + stmtBuilder.setInt(i.get(), config.getDefaultTtl()); } return getFuture(executeAsyncWrite(ctx, stmtBuilder.build()), rs -> null); } @@ -267,9 +267,9 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { boolean hasChanges = false; switch (fromVersion) { case 0: - if (!oldConfiguration.has("defaultTtL")) { + if (!oldConfiguration.has("defaultTtl")) { hasChanges = true; - ((ObjectNode) oldConfiguration).put("defaultTtL", 0); + ((ObjectNode) oldConfiguration).put("defaultTtl", 0); } break; default: diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java index e391e178f5..fefe871f82 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeConfiguration.java @@ -27,13 +27,13 @@ public class TbSaveToCustomCassandraTableNodeConfiguration implements NodeConfig private String tableName; private Map fieldsMapping; - private int defaultTtL; + private int defaultTtl; @Override public TbSaveToCustomCassandraTableNodeConfiguration defaultConfiguration() { TbSaveToCustomCassandraTableNodeConfiguration configuration = new TbSaveToCustomCassandraTableNodeConfiguration(); - configuration.setDefaultTtL(0); + configuration.setDefaultTtl(0); configuration.setTableName(""); Map map = new HashMap<>(); map.put("", ""); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java index 2ef5d4173c..7df2a982de 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNodeTest.java @@ -135,7 +135,7 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad public void verifyDefaultConfig() { assertThat(config.getTableName()).isEqualTo(""); assertThat(config.getFieldsMapping()).isEqualTo(Map.of("", "")); - assertThat(config.getDefaultTtL()).isEqualTo(0); + assertThat(config.getDefaultTtl()).isEqualTo(0); } @Test @@ -240,7 +240,7 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad Consumer verifyBuilder) throws TbNodeException { config.setTableName("readings"); config.setFieldsMapping(Map.of("$entityId", "entityIdTableColumn")); - config.setDefaultTtL(ttlFromConfig); + config.setDefaultTtl(ttlFromConfig); mockOnInit(); willAnswer(invocation -> boundStatementBuilderMock).given(node).getStmtBuilder(); @@ -271,7 +271,7 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad @Test public void givenValidMsgStructure_whenOnMsg_thenVerifyMatchOfValuesInsertionOrderIntoStatementAndSaveToCustomCassandraTable() throws TbNodeException { - config.setDefaultTtL(25); + config.setDefaultTtl(25); config.setTableName("readings"); Map mappings = Map.of( "$entityId", "entityIdTableColumn", @@ -325,13 +325,13 @@ public class TbSaveToCustomCassandraTableNodeTest extends AbstractRuleNodeUpgrad Arguments.of(0, "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"}}", true, - "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTtL\":0}" + "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTtl\":0}" ), // default config for version 1 with upgrade from version 1 Arguments.of(1, - "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTtL\":0}", + "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTtl\":0}", false, - "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTtL\":0}" + "{\"tableName\":\"\",\"fieldsMapping\":{\"\":\"\"},\"defaultTtl\":0}" ) ); } From 434625c47df6257a5d77ec09cebb6a1c01d59f60 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 27 Nov 2024 16:44:53 +0200 Subject: [PATCH 34/37] fixed not-null check --- .../thingsboard/server/controller/MobileAppController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/MobileAppController.java b/application/src/main/java/org/thingsboard/server/controller/MobileAppController.java index 3626ff4c3e..362539f6b6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/MobileAppController.java +++ b/application/src/main/java/org/thingsboard/server/controller/MobileAppController.java @@ -82,9 +82,9 @@ public class MobileAppController extends BaseController { public LoginMobileInfo getLoginMobileInfo(@Parameter(description = "Mobile application package name") @RequestParam String pkgName, @Parameter(description = "Platform type", schema = @Schema(allowableValues = {"ANDROID", "IOS"})) - @RequestParam PlatformType platform) throws ThingsboardException { + @RequestParam PlatformType platform) { List oauth2Clients = oAuth2ClientService.findOAuth2ClientLoginInfosByMobilePkgNameAndPlatformType(pkgName, platform); - MobileApp mobileApp = checkNotNull(mobileAppService.findMobileAppByPkgNameAndPlatformType(pkgName, platform)); + MobileApp mobileApp = mobileAppService.findMobileAppByPkgNameAndPlatformType(pkgName, platform); StoreInfo storeInfo = Optional.ofNullable(mobileApp).map(MobileApp::getStoreInfo).orElse(null); MobileAppVersionInfo versionInfo = Optional.ofNullable(mobileApp).map(MobileApp::getVersionInfo).orElse(null); return new LoginMobileInfo(oauth2Clients, storeInfo, versionInfo); From eedbaeddaf74c90b7a1581349615dd15aa17bdaa Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 27 Nov 2024 17:41:37 +0200 Subject: [PATCH 35/37] Improvements for resources upgrade --- .../install/update/ResourcesUpdater.java | 68 ++++++++++++------- .../csv/AbstractBulkImportService.java | 19 ++---- .../common/util/ThingsBoardExecutors.java | 15 ++++ 3 files changed, 66 insertions(+), 36 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/ResourcesUpdater.java b/application/src/main/java/org/thingsboard/server/service/install/update/ResourcesUpdater.java index eb3760cac6..00797da449 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/ResourcesUpdater.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/ResourcesUpdater.java @@ -16,10 +16,12 @@ package org.thingsboard.server.service.install.update; import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.HasImage; import org.thingsboard.server.common.data.id.DashboardId; @@ -42,6 +44,8 @@ import org.thingsboard.server.dao.widget.WidgetTypeDao; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleDao; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; import java.util.function.Function; @@ -86,44 +90,60 @@ public class ResourcesUpdater { log.debug("Created/updated system images and resources for default dashboard '{}'", defaultDashboard.getTitle()); } + @SneakyThrows public void updateDashboardsResources() { log.info("Updating resources usage in dashboards"); + var executor = ThingsBoardExecutors.newLimitedTasksExecutor(4, 4, "dashboards-resources-upgrade"); + var dashboards = new PageDataIterable<>(dashboardService::findAllDashboardsIds, 512); - int totalCount = 0; - int updatedCount = 0; + AtomicInteger totalCount = new AtomicInteger(); + AtomicInteger updatedCount = new AtomicInteger(); for (DashboardId dashboardId : dashboards) { - Dashboard dashboard = dashboardService.findDashboardById(TenantId.SYS_TENANT_ID, dashboardId); - boolean updated = resourceService.updateResourcesUsage(dashboard); // will convert resources ids to new structure - if (updated) { - dashboardService.saveDashboard(dashboard); - updatedCount++; - } - totalCount++; + executor.submit(() -> { + Dashboard dashboard = dashboardService.findDashboardById(TenantId.SYS_TENANT_ID, dashboardId); + boolean updated = resourceService.updateResourcesUsage(dashboard); // will convert resources ids to new structure + if (updated) { + dashboardService.saveDashboard(dashboard); + updatedCount.incrementAndGet(); + } + if (totalCount.incrementAndGet() % 1000 == 0) { + log.info("Processed {} dashboards, updated {}", totalCount, updatedCount); + } + }); + } - if (totalCount % 1000 == 0) { - log.info("Processed {} dashboards, updated {}", totalCount, updatedCount); - } + executor.shutdown(); + if (!executor.awaitTermination(10, TimeUnit.MINUTES)) { + throw new RuntimeException("Dashboards resources update timeout"); // just in case, should happen } log.info("Updated {} dashboards", updatedCount); } + @SneakyThrows public void updateWidgetsResources() { log.info("Updating resources usage in widgets"); - int totalCount = 0; - int updatedCount = 0; + var executor = ThingsBoardExecutors.newLimitedTasksExecutor(4, 4, "widgets-resources-upgrade"); + + AtomicInteger totalCount = new AtomicInteger(); + AtomicInteger updatedCount = new AtomicInteger(); var widgets = new PageDataIterable<>(widgetTypeService::findAllWidgetTypesIds, 512); for (WidgetTypeId widgetTypeId : widgets) { - WidgetTypeDetails widgetTypeDetails = widgetTypeService.findWidgetTypeDetailsById(TenantId.SYS_TENANT_ID, widgetTypeId); - boolean updated = resourceService.updateResourcesUsage(widgetTypeDetails); - if (updated) { - widgetTypeService.saveWidgetType(widgetTypeDetails); - updatedCount++; - } - totalCount++; + executor.submit(() -> { + WidgetTypeDetails widgetTypeDetails = widgetTypeService.findWidgetTypeDetailsById(TenantId.SYS_TENANT_ID, widgetTypeId); + boolean updated = resourceService.updateResourcesUsage(widgetTypeDetails); + if (updated) { + widgetTypeService.saveWidgetType(widgetTypeDetails); + updatedCount.incrementAndGet(); + } + if (totalCount.incrementAndGet() % 200 == 0) { + log.info("Processed {} widgets, updated {}", totalCount, updatedCount); + } + }); + } - if (totalCount % 200 == 0) { - log.info("Processed {} widgets, updated {}", totalCount, updatedCount); - } + executor.shutdown(); + if (!executor.awaitTermination(10, TimeUnit.MINUTES)) { + throw new RuntimeException("Widgets resources update timeout"); } log.info("Updated {} widgets", updatedCount); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java index ff764b7db3..50d8b9c371 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java @@ -31,7 +31,7 @@ import org.springframework.security.core.context.SecurityContext; import org.springframework.security.core.context.SecurityContextHolder; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; @@ -71,8 +71,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -90,16 +89,11 @@ public abstract class AbstractBulkImportService(150_000), - ThingsBoardThreadFactory.forName("bulk-import"), new ThreadPoolExecutor.CallerRunsPolicy()); - executor.allowCoreThreadTimeOut(true); - } + executor = ThingsBoardExecutors.newLimitedTasksExecutor(Runtime.getRuntime().availableProcessors(), 150_000, "bulk-import"); } public final BulkImportResult processBulkImport(BulkImportRequest request, SecurityUser user) throws Exception { @@ -287,8 +281,8 @@ public abstract class AbstractBulkImportService fields = new LinkedHashMap<>(); private final Map kvs = new LinkedHashMap<>(); private int lineNumber; + } @Data diff --git a/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardExecutors.java b/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardExecutors.java index fde2362091..913f32c7f9 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardExecutors.java +++ b/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardExecutors.java @@ -17,6 +17,9 @@ package org.thingsboard.common.util; import java.util.concurrent.ExecutorService; import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; public class ThingsBoardExecutors { @@ -48,4 +51,16 @@ public class ThingsBoardExecutors { return newWorkStealingPool(parallelism, clazz.getSimpleName()); } + /* + * executor with limited tasks queue size + * */ + public static ExecutorService newLimitedTasksExecutor(int threads, int maxQueueSize, String name) { + ThreadPoolExecutor executor = new ThreadPoolExecutor(threads, threads, + 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(maxQueueSize), + ThingsBoardThreadFactory.forName(name), + new ThreadPoolExecutor.CallerRunsPolicy()); + executor.allowCoreThreadTimeOut(true); + return executor; + } + } From 28cf8c8868f54c81e30f89f86956ab9461363984 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 27 Nov 2024 18:10:11 +0200 Subject: [PATCH 36/37] Update cache cleanup service --- .../service/install/update/DefaultCacheCleanupService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java index 8110642d7b..1542133764 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java @@ -65,6 +65,10 @@ public class DefaultCacheCleanupService implements CacheCleanupService { log.info("Clearing cache to upgrade from version 3.7.0 to 3.8.0"); clearAll(); break; + case "3.8.1": + log.info("Clearing cache to upgrade from version 3.8.1 to 3.9.0"); + clearAll(); + break; default: //Do nothing, since cache cleanup is optional. } From 332b4b57cb4c0a8910091e7ca5ed592fa7a7afc0 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 28 Nov 2024 17:14:08 +0200 Subject: [PATCH 37/37] Remove gitter from readme --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index c9e6e17fc2..6bce2db844 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,6 @@ Collect and Visualize your IoT data in minutes by following this [guide](https:/ ## Support - - [Community chat](https://gitter.im/thingsboard/chat) - [Q&A forum](https://groups.google.com/forum/#!forum/thingsboard) - [Stackoverflow](http://stackoverflow.com/questions/tagged/thingsboard)