diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java index 0469b45442..f879fb21c7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java @@ -17,9 +17,15 @@ package org.thingsboard.server.dao.sql.query; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.core.SqlParameter; +import org.springframework.jdbc.core.SqlParameterValue; +import org.springframework.jdbc.core.namedparam.NamedParameterUtils; +import org.springframework.jdbc.core.namedparam.ParsedSql; +import org.springframework.jdbc.core.namedparam.SqlParameterSource; import org.springframework.stereotype.Component; -import java.util.Arrays; +import java.util.List; +import java.util.UUID; @Component @Slf4j @@ -33,8 +39,72 @@ public class DefaultQueryLogComponent implements QueryLogComponent { @Override public void logQuery(QueryContext ctx, String query, long duration) { if (logSqlQueries && duration > logQueriesThreshold) { - log.info("QUERY: {} took {} ms", query, duration); - Arrays.asList(ctx.getParameterNames()).forEach(param -> log.info("QUERY PARAM: {} -> {}", param, ctx.getValue(param))); + + String sqlToUse = substituteParametersInSqlString(query, ctx); + log.warn("SLOW QUERY took {} ms: {}", duration, sqlToUse); + + } + } + + String substituteParametersInSqlString(String sql, SqlParameterSource paramSource) { + + ParsedSql parsedSql = NamedParameterUtils.parseSqlStatement(sql); + List declaredParams = NamedParameterUtils.buildSqlParameterList(parsedSql, paramSource); + + if (declaredParams.isEmpty()) { + return sql; + } + + for (SqlParameter parSQL: declaredParams) { + String paramName = parSQL.getName(); + if (!paramSource.hasValue(paramName)) { + continue; + } + + Object value = paramSource.getValue(paramName); + if (value instanceof SqlParameterValue) { + value = ((SqlParameterValue)value).getValue(); + } + + if (!(value instanceof Iterable)) { + + String ValueForSQLQuery = getValueForSQLQuery(value); + sql = sql.replace(":" + paramName, ValueForSQLQuery); + continue; + } + + //Iterable + int count = 0; + String valueArrayStr = ""; + + for (Object valueTemp: (Iterable)value) { + + if (count > 0) { + valueArrayStr+=", "; + } + + String valueForSQLQuery = getValueForSQLQuery(valueTemp); + valueArrayStr += valueForSQLQuery; + ++count; + } + + sql = sql.replace(":" + paramName, valueArrayStr); + } + + return sql; + } + + String getValueForSQLQuery(Object valueParameter) { + + if (valueParameter instanceof String) { + return "'" + ((String) valueParameter).replaceAll("'", "''") + "'"; + } + + if (valueParameter instanceof UUID) { + return "'" + valueParameter + "'"; + } + + return String.valueOf(valueParameter); } } diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 4e2596485b..2930ac7414 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -242,10 +242,17 @@ CREATE TABLE IF NOT EXISTS device_profile ( CONSTRAINT fk_software_device_profile FOREIGN KEY (software_id) REFERENCES ota_package(id) ); -ALTER TABLE ota_package - ADD CONSTRAINT fk_device_profile_ota_package - FOREIGN KEY (device_profile_id) REFERENCES device_profile (id) - ON DELETE CASCADE; +DO +$$ + BEGIN + IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'fk_device_profile_ota_package') THEN + ALTER TABLE ota_package + ADD CONSTRAINT fk_device_profile_ota_package + FOREIGN KEY (device_profile_id) REFERENCES device_profile (id) + ON DELETE CASCADE; + END IF; + END; +$$; -- We will use one-to-many relation in the first release and extend this feature in case of user requests -- CREATE TABLE IF NOT EXISTS device_profile_firmware ( diff --git a/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java index bc7abf8342..12f77ece14 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java +++ b/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java @@ -25,6 +25,7 @@ import org.junit.runner.RunWith; "org.thingsboard.server.dao.service.event.sql.*SqlTest", "org.thingsboard.server.dao.service.sql.*SqlTest", "org.thingsboard.server.dao.service.timeseries.sql.*SqlTest", + "org.thingsboard.server.dao.service.install.sql.*SqlTest" }) public class SqlDaoServiceTestSuite { } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/install/sql/EntitiesSchemaSqlTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/install/sql/EntitiesSchemaSqlTest.java new file mode 100644 index 0000000000..285ea5bb50 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/install/sql/EntitiesSchemaSqlTest.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.install.sql; + +import org.assertj.core.api.Assertions; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.core.JdbcTemplate; +import org.thingsboard.server.dao.service.AbstractServiceTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +@DaoSqlTest +public class EntitiesSchemaSqlTest extends AbstractServiceTest { + + @Value("${classpath:sql/schema-entities.sql}") + private Path installScriptPath; + + @Autowired + private JdbcTemplate jdbcTemplate; + + @Test + public void testRepeatedInstall() throws IOException { + String installScript = Files.readString(installScriptPath); + try { + for (int i = 1; i <= 2; i++) { + jdbcTemplate.execute(installScript); + } + } catch (Exception e) { + Assertions.fail("Failed to execute reinstall", e); + } + } + + @Test + public void testRepeatedInstall_badScript() { + String illegalInstallScript = "CREATE TABLE IF NOT EXISTS qwerty ();\n" + + "ALTER TABLE qwerty ADD COLUMN first VARCHAR(10);"; + + assertDoesNotThrow(() -> { + jdbcTemplate.execute(illegalInstallScript); + }); + + try { + assertThatThrownBy(() -> { + jdbcTemplate.execute(illegalInstallScript); + }).getCause().hasMessageContaining("column").hasMessageContaining("already exists"); + } finally { + jdbcTemplate.execute("DROP TABLE qwerty;"); + } + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java new file mode 100644 index 0000000000..c3ac1f2c24 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java @@ -0,0 +1,154 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.query; + +import org.junit.Before; +import org.junit.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.runner.RunWith; +import org.mockito.BDDMockito; +import org.mockito.Mockito; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.test.context.junit4.SpringRunner; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.List; +import java.util.UUID; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.times; + +@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = DefaultQueryLogComponent.class) +@EnableConfigurationProperties +@TestPropertySource(properties = { + "sql.log_queries=true", + "sql.log_queries_threshold:2999" +}) + +public class DefaultQueryLogComponentTest { + + private TenantId tenantId; + private QueryContext ctx; + + @SpyBean + private DefaultQueryLogComponent queryLog; + + @Before + public void setUp() { + tenantId = new TenantId(UUID.fromString("97275c1c-9cf2-4d25-a68d-933031158f84")); + ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); + } + + @Test + public void logQuery() { + + BDDMockito.willReturn("").given(queryLog).substituteParametersInSqlString("", ctx); + queryLog.logQuery(ctx, "", 3000); + + Mockito.verify(queryLog, times(1)).substituteParametersInSqlString("", ctx); + + } + + @Test + public void substituteParametersInSqlString_StringType() { + + String sql = "Select * from Table Where name = :name AND id = :id"; + String sqlToUse = "Select * from Table Where name = 'Mery''s' AND id = 'ID_1'"; + + ctx.addStringParameter("name", "Mery's"); + ctx.addStringParameter("id", "ID_1"); + + String sqlToUseResult = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUseResult); + } + + @Test + public void substituteParametersInSqlString_DoubleLongType() { + + double sum = 0.00000021d; + long price = 100000; + String sql = "Select * from Table Where sum = :sum AND price = :price"; + String sqlToUse = "Select * from Table Where sum = 2.1E-7 AND price = 100000"; + + ctx.addDoubleParameter("sum", sum); + ctx.addLongParameter("price", price); + + String sqlToUseResult = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUseResult); + } + + @Test + public void substituteParametersInSqlString_BooleanType() { + + String sql = "Select * from Table Where check = :check AND mark = :mark"; + String sqlToUse = "Select * from Table Where check = true AND mark = false"; + + ctx.addBooleanParameter("check", true); + ctx.addBooleanParameter("mark", false); + + String sqlToUseResult = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUseResult); + } + + @Test + public void substituteParametersInSqlString_UuidType() { + + UUID guid = UUID.randomUUID(); + String sql = "Select * from Table Where guid = :guid"; + String sqlToUse = "Select * from Table Where guid = '" + guid + "'"; + + ctx.addUuidParameter("guid", guid); + + String sqlToUseResult = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUseResult); + } + + @Test + public void substituteParametersInSqlString_StringListType() { + + List ids = List.of("ID_1'", "ID_2", "ID_3", "ID_4"); + + String sql = "Select * from Table Where id IN (:ids)"; + String sqlToUse = "Select * from Table Where id IN ('ID_1''', 'ID_2', 'ID_3', 'ID_4')"; + + ctx.addStringListParameter("ids", ids); + + String sqlToUseResult = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUseResult); + } + + @Test + public void substituteParametersInSqlString_UuidListType() { + + List guids = List.of(UUID.fromString("634a8d03-6871-4e01-94d0-876bf3e67dff"), UUID.fromString("3adbb5b8-4dc6-4faf-80dc-681a7b518b5e"), UUID.fromString("63a50f0c-2058-4d1d-8f15-812eb7f84412")); + + String sql = "Select * from Table Where guid IN (:guids)"; + String sqlToUse = "Select * from Table Where guid IN ('634a8d03-6871-4e01-94d0-876bf3e67dff', '3adbb5b8-4dc6-4faf-80dc-681a7b518b5e', '63a50f0c-2058-4d1d-8f15-812eb7f84412')"; + + ctx.addUuidListParameter("guids", guids); + + String sqlToUseResult = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUseResult); + } +} + diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java index 5acf944c98..836497cced 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java @@ -158,7 +158,7 @@ public class TbKafkaNode implements TbNode { } private void processRecord(TbContext ctx, TbMsg msg, RecordMetadata metadata, Exception e) { - if (metadata != null) { + if (e == null) { TbMsg next = processResponse(ctx, msg, metadata); ctx.tellNext(next, TbRelationTypes.SUCCESS); } else { diff --git a/ui-ngx/.yarnrc b/ui-ngx/.yarnrc new file mode 100644 index 0000000000..a3a3d23ec6 --- /dev/null +++ b/ui-ngx/.yarnrc @@ -0,0 +1 @@ +network-timeout 1000000 diff --git a/ui-ngx/generate-types.js b/ui-ngx/generate-types.js index a24673cbd5..68b31d6568 100644 --- a/ui-ngx/generate-types.js +++ b/ui-ngx/generate-types.js @@ -17,9 +17,12 @@ const child_process = require("child_process"); const fs = require('fs'); const path = require('path'); -const typeDir = './target/types'; -const srcDir = typeDir + '/src'; -const moduleMapPath = "src/app/modules/common/modules-map.ts"; + +const typeDir = path.join('.', 'target', 'types'); +const srcDir = path.join('.', 'target', 'types', 'src'); +const moduleMapPath = path.join('src', 'app', 'modules', 'common', 'modules-map.ts'); +const ngcPath = path.join('.', 'node_modules', '.bin', 'ngc'); +const tsconfigPath = path.join('src', 'tsconfig.app.json'); console.log(`Remove directory: ${typeDir}`); try { @@ -28,7 +31,7 @@ try { console.error(`Remove directory error: ${err}`); } -const cliCommand = `./node_modules/.bin/ngc --p src/tsconfig.app.json --declaration --outDir ${srcDir}`; +const cliCommand = `${ngcPath} --p ${tsconfigPath} --declaration --outDir ${srcDir}`; console.log(cliCommand); try { child_process.execSync(cliCommand);