diff --git a/dao/pom.xml b/dao/pom.xml index 922b389bff..1488d7eec5 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -201,6 +201,11 @@ org.springframework.boot spring-boot-starter-data-jpa + + org.springframework.boot + spring-boot-starter-test + test + org.springframework spring-test @@ -211,6 +216,16 @@ hsqldb test + + org.testcontainers + postgresql + test + + + org.testcontainers + jdbc + test + org.springframework spring-context-support diff --git a/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java new file mode 100644 index 0000000000..5d6f1a374f --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java @@ -0,0 +1,76 @@ +/** + * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * + * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of ThingsBoard, Inc. and its suppliers, + * if any. The intellectual and technical concepts contained + * herein are proprietary to ThingsBoard, Inc. + * and its suppliers and may be covered by U.S. and Foreign Patents, + * patents in process, and are protected by trade secret or copyright law. + * + * Dissemination of this information or reproduction of this material is strictly forbidden + * unless prior written permission is obtained from COMPANY. + * + * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, + * managers or contractors who have executed Confidentiality and Non-disclosure agreements + * explicitly covering such access. + * + * The copyright notice above does not evidence any actual or intended publication + * or disclosure of this source code, which includes + * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. + * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, + * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT + * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, + * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. + * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION + * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, + * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + */ +package org.thingsboard.server.dao; + +import org.junit.ClassRule; +import org.junit.extensions.cpsuite.ClasspathSuite; +import org.junit.extensions.cpsuite.ClasspathSuite.ClassnameFilters; +import org.junit.runner.RunWith; + +import java.util.Arrays; + +@RunWith(ClasspathSuite.class) +@ClassnameFilters({ + "org.thingsboard.server.dao.service.psql.*SqlTest", + "org.thingsboard.server.dao.service.attributes.psql.*SqlTest", + "org.thingsboard.server.dao.service.event.psql.*SqlTest", + "org.thingsboard.server.dao.service.timeseries.psql.*SqlTest" +}) +public class PostgreSqlDaoServiceTestSuite { + + @ClassRule + public static CustomSqlUnit sqlUnit = new CustomSqlUnit( + Arrays.asList("sql/schema-ts-psql.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql" + , "sql/system-data.sql" + , "sql/system-test-psql.sql" + ), + "sql/psql/drop-all-tables.sql", + "psql-test.properties" + ); + +// @ClassRule +// public static CustomSqlUnit sqlUnit = new CustomSqlUnit( +// Arrays.asList("sql/schema-ts-psql.sql" +// , "sql/schema-entities.sql", "sql/schema-entities-idx.sql" +// , "sql/system-data.sql", "sql/system-test.sql" +// ), +// "sql/psql/drop-all-tables.sql", +// "sql-test.properties" +// ); + +// @ClassRule +// public static CustomSqlUnit sqlUnit = new CustomSqlUnit( +// Arrays.asList("sql/schema-timescale.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"), +// "sql/timescale/drop-all-tables.sql", +// "sql-test.properties" +// ); + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index 357af4e644..fbace01ddb 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -17,15 +17,18 @@ package org.thingsboard.server.dao.service; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; import org.hamcrest.Matchers; +import org.jetbrains.annotations.NotNull; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.ResultSetExtractor; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; @@ -80,14 +83,17 @@ import java.util.Comparator; import java.util.List; import java.util.Random; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.Assert.assertEquals; import static org.hamcrest.MatcherAssert.assertThat; +@Slf4j public abstract class BaseEntityServiceTest extends AbstractServiceTest { + static final int ENTITY_COUNT = 5; @Autowired private AttributesService attributesService; @@ -525,7 +531,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } private void createTestHierarchy(TenantId tenantId, List assets, List devices, List consumptions, List highConsumptions, List temperatures, List highTemperatures) throws InterruptedException { - for (int i = 0; i < 5; i++) { + for (int i = 0; i < ENTITY_COUNT; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); asset.setName("Asset" + i); @@ -535,18 +541,19 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { //TO make sure devices have different created time Thread.sleep(1); assets.add(asset); - EntityRelation er = new EntityRelation(); - er.setFrom(tenantId); - er.setTo(asset.getId()); - er.setType("Manages"); - er.setTypeGroup(RelationTypeGroup.COMMON); - relationService.saveRelation(tenantId, er); + createRelation(tenantId, "Manages", tenantId, asset.getId()); long consumption = (long) (Math.random() * 100); consumptions.add(consumption); if (consumption > 50) { highConsumptions.add(consumption); } - for (int j = 0; j < 5; j++) { + + //tenant -> asset : one-to-one but many edges + for (int n = 0; n < ENTITY_COUNT; n++) { + createRelation(tenantId, "UseCase-" + n, tenantId, asset.getId()); + } + + for (int j = 0; j < ENTITY_COUNT; j++) { Device device = new Device(); device.setTenantId(tenantId); device.setName("A" + i + "Device" + j); @@ -556,27 +563,158 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { //TO make sure devices have different created time Thread.sleep(1); devices.add(device); - er = new EntityRelation(); - er.setFrom(asset.getId()); - er.setTo(device.getId()); - er.setType("Contains"); - er.setTypeGroup(RelationTypeGroup.COMMON); - relationService.saveRelation(tenantId, er); + createRelation(tenantId, "Contains", asset.getId(), device.getId()); long temperature = (long) (Math.random() * 100); temperatures.add(temperature); if (temperature > 45) { highTemperatures.add(temperature); } + + //asset -> device : one-to-one but many edges + for (int n = 0; n < ENTITY_COUNT; n++) { + createRelation(tenantId, "UseCase-" + n, asset.getId(), device.getId()); + } } } - createManyCustomRelationsBetweenTwoNodes(tenantId, "UseCase", assets, devices); + //asset -> device one-to-many shared with other assets + for (int n = 0; n < devices.size(); n = n + ENTITY_COUNT) { + createRelation(tenantId, "SharedWithAsset0", assets.get(0).getId(), devices.get(n).getId()); + } + + //createManyCustomRelationsBetweenTwoNodes(tenantId, "UseCase", assets, devices); createHorizontalRingRelations(tenantId, "Ring(Loop)-Ast", assets); createLoopRelations(tenantId, "Loop-Tnt-Ast-Dev", tenantId, assets.get(0).getId(), devices.get(0).getId()); createLoopRelations(tenantId, "Loop-Tnt-Ast", tenantId, assets.get(1).getId()); createLoopRelations(tenantId, "Loop-Ast-Tnt-Ast", assets.get(2).getId(), tenantId, assets.get(3).getId()); - printAllRelations(); + //testQuery(); + + //printAllRelations(); + } + + private void testQuery() { + + template.query("" + + " DROP TABLE IF EXISTS relation_test;\n" + + " CREATE TABLE IF NOT EXISTS relation_test\n" + + " (\n" + + " from_id uuid,\n" + + " from_type varchar(255),\n" + + " to_id uuid,\n" + + " to_type varchar(255),\n" + + " relation_type_group varchar(255),\n" + + " relation_type varchar(255),\n" + + " additional_info varchar,\n" + + " CONSTRAINT relation_test_pkey PRIMARY KEY (from_id, from_type, relation_type_group, relation_type, to_id, to_type)\n" + + " );\n", r -> null); + + log.error("insert test {}", (Object) template.query("" + + "INSERT INTO relation_test (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info) " + + " VALUES " + + " ('11111111-0f19-11ec-ba23-e981fc95500d', 'TENANT', '22222222-0f19-11ec-ba23-e981fc95500d', 'ASSET', 'COMMON', 'Contains', null),\n" + + " ('22222222-0f19-11ec-ba23-e981fc95500d', 'ASSET', '33333333-0f19-11ec-ba23-e981fc95500d', 'DEVICE', 'COMMON', 'Contains', null),\n" + + " ('33333333-0f19-11ec-ba23-e981fc95500d', 'DEVICE', '11111111-0f19-11ec-ba23-e981fc95500d', 'TENANT', 'COMMON', 'Contains', null);\n" + + "" + , r -> null)); + + + //log.error("array_position hsql {}", template.queryForObject("select array_position(ARRAY['sun','mon','tue','wed','thu','fri','sat'], 'mon') from relation_test", Long.class)); + + log.error("array_position hsql {}", template.queryForObject("select position_array('mon' in ARRAY['sun','mon','tue']);", Long.class)); + log.error("array_position psql {}", template.queryForObject("select array_position(ARRAY['sun','mon','tue'], 'mon');", Long.class)); + + Long countTest = template.queryForObject("select count(*) from relation_test", Long.class); + log.error("count test {}", countTest); + + Long count = template.queryForObject("select count(*) from relation", Long.class); + log.error("count {}", count); + + List> result = template.query("" + + "WITH RECURSIVE related_entities(from_id, from_type, to_id, to_type, lvl, path) " + + " AS (SELECT from_id, " + + " from_type, " + + " to_id, " + + " to_type, " + + " 1 as lvl, " + + " ARRAY [from_id] as path " + + " FROM relation_test r " + + " WHERE from_id = '11111111-0f19-11ec-ba23-e981fc95500d' " + + " and from_type = 'TENANT' " + + " and relation_type_group = 'COMMON' " + + " GROUP BY r.from_id, r.from_type, r.to_id, r.to_type , 1, ARRAY [from_id] " + + " UNION ALL " + + " SELECT r.from_id, " + + " r.from_type, " + + " r.to_id, " + + " r.to_type, " + + " (re.lvl + 1) as lvl, " + + " (re.path || ARRAY [r.from_id]) as path " + + " FROM relation_test r " + + " INNER JOIN related_entities re " + + " ON r.from_id = re.to_id and " + + " r.from_type = re.to_type and " + + " relation_type_group = 'COMMON' and " + + " r.from_id NOT IN (SELECT * FROM unnest(re.path)) and " + + " re.lvl <= 7 " + + " GROUP BY r.from_id, r.from_type, r.to_id, r.to_type, " + + " (re.lvl + 1), (re.path || ARRAY [r.from_id])) " + + " " + + " SELECT lvl, from_id, from_type, to_id, to_type, path " + //to_id IN (SELECT * FROM unnest(path)) as is_present, + " from related_entities r_int " + + " " + + " ", + getListResultSetExtractor()); + + log.error("result {}", result.size() - 1); + AtomicInteger counter = new AtomicInteger(); + result.forEach(r -> System.out.printf("%s %s\n", counter.incrementAndGet(), r.toString())); + log.error("end"); + + log.error("query 1 (expected true): {}", template.queryForObject( + "SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3') NOT IN (SELECT * FROM unnest(ARRAY[UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3'), UUID('46957d30-0f38-11ec-8153-55a9f38b54f3')])) ", + String.class)); + log.error("query 1.1 (expected true): {}", template.queryForObject( + "SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3') NOT IN (SELECT * FROM unnest(ARRAY[UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3')] || ARRAY[UUID('46957d30-0f38-11ec-8153-55a9f38b54f3')] )) ", + String.class)); + log.error("query 2 (expected true): {}", template.queryForObject( + "SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3') NOT IN (SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3')) ", + String.class)); +// log.error("query true3: ", template.queryForObject( +// "SELECT '463e5c80-0f38-11ec-8153-55a9f38b54f3' IN (SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3')) ", +// String.class)); + log.error("query 3 (expected true): {} ", template.queryForObject( + "SELECT '463e5c80-0f38-11ec-8153-55a9f38b54f3' NOT IN (SELECT '463e5c80-0f38-11ec-8153-55a9f38b54f3') ", + String.class)); + + List> result2 = template.query("SELECT * FROM unnest(ARRAY[UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3'), UUID('46957d30-0f38-11ec-8153-55a9f38b54f3')]) " + , getListResultSetExtractor()); + log.error("result2 {}", result2.size() - 1); + AtomicInteger counter2 = new AtomicInteger(); + result2.forEach(r -> System.out.printf("%s %s\n", counter2.incrementAndGet(), r.toString())); + log.error("end2"); + + } + + @NotNull + private ResultSetExtractor>> getListResultSetExtractor() { + return rs -> { + List> list = new ArrayList<>(); + final int columnCount = rs.getMetaData().getColumnCount(); + List columns = new ArrayList<>(columnCount); + for (int i = 1; i <= columnCount; i++) { + columns.add(rs.getMetaData().getColumnName(i)); + } + list.add(columns); + while (rs.next()) { + List data = new ArrayList<>(columnCount); + for (int i = 1; i <= columnCount; i++) { + data.add(rs.getString(i)); + } + list.add(data); + } + return list; + }; } /* @@ -642,9 +780,14 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } void createOneToManyRelations(TenantId tenantId, String type, EntityId from, List toIds) { - toIds.forEach(toId -> relationService.saveRelation(tenantId, new EntityRelation(from, toId, type, RelationTypeGroup.COMMON))); + toIds.forEach(toId -> createRelation(tenantId, type, from, toId)); + } + + void createRelation(TenantId tenantId, String type, EntityId from, EntityId toId) { + relationService.saveRelation(tenantId, new EntityRelation(from, toId, type, RelationTypeGroup.COMMON)); } + @Test public void testSimpleFindEntityDataByQuery() throws InterruptedException { List devices = new ArrayList<>(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DaoPostgreSqlTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DaoPostgreSqlTest.java new file mode 100644 index 0000000000..16da531369 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DaoPostgreSqlTest.java @@ -0,0 +1,48 @@ +/** + * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * + * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of ThingsBoard, Inc. and its suppliers, + * if any. The intellectual and technical concepts contained + * herein are proprietary to ThingsBoard, Inc. + * and its suppliers and may be covered by U.S. and Foreign Patents, + * patents in process, and are protected by trade secret or copyright law. + * + * Dissemination of this information or reproduction of this material is strictly forbidden + * unless prior written permission is obtained from COMPANY. + * + * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, + * managers or contractors who have executed Confidentiality and Non-disclosure agreements + * explicitly covering such access. + * + * The copyright notice above does not evidence any actual or intended publication + * or disclosure of this source code, which includes + * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. + * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, + * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT + * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, + * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. + * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION + * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, + * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + */ +package org.thingsboard.server.dao.service; + +import org.springframework.test.context.TestPropertySource; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Inherited +@Documented +@TestPropertySource(locations = {"classpath:application-test.properties", "classpath:psql-test.properties"}) +public @interface DaoPostgreSqlTest { +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/sql/EntityServiceSqlTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/psql/EntityServicePostgreSqlTest.java similarity index 77% rename from dao/src/test/java/org/thingsboard/server/dao/service/sql/EntityServiceSqlTest.java rename to dao/src/test/java/org/thingsboard/server/dao/service/psql/EntityServicePostgreSqlTest.java index 0c59ae49c8..5b8c6d4aca 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/sql/EntityServiceSqlTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/psql/EntityServicePostgreSqlTest.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.service.sql; +package org.thingsboard.server.dao.service.psql; import org.thingsboard.server.dao.service.BaseEntityServiceTest; -import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.service.DaoPostgreSqlTest; -@DaoSqlTest -public class EntityServiceSqlTest extends BaseEntityServiceTest { +@DaoPostgreSqlTest +public class EntityServicePostgreSqlTest extends BaseEntityServiceTest { } diff --git a/dao/src/test/resources/psql-test.properties b/dao/src/test/resources/psql-test.properties new file mode 100644 index 0000000000..00420be628 --- /dev/null +++ b/dao/src/test/resources/psql-test.properties @@ -0,0 +1,47 @@ +database.ts.type=sql +database.ts_latest.type=sql +sql.ts_inserts_executor_type=fixed +sql.ts_inserts_fixed_thread_pool_size=200 +sql.ts_key_value_partitioning=MONTHS +# +spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true +spring.jpa.properties.hibernate.order_by.default_null_ordering=last +spring.jpa.properties.hibernate.jdbc.log.warnings=false +spring.jpa.show-sql=false +spring.jpa.hibernate.ddl-auto=none +spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect +spring.datasource.username=postgres +spring.datasource.password=postgres +spring.datasource.url=jdbc:tc:postgresql:12.8:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw +spring.datasource.driverClassName=org.testcontainers.jdbc.ContainerDatabaseDriver +#org.postgresql.Driver +spring.datasource.hikari.maximumPoolSize=20 +service.type=monolith +#database.ts.type=timescale +#database.ts.type=sql +#database.entities.type=sql +# +#sql.ts_inserts_executor_type=fixed +#sql.ts_inserts_fixed_thread_pool_size=200 +#sql.ts_key_value_partitioning=MONTHS +# +#spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true +#spring.jpa.show-sql=false +#spring.jpa.hibernate.ddl-auto=none +#spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect +# +#spring.datasource.username=postgres +#spring.datasource.password=postgres +#spring.datasource.url=jdbc:postgresql://localhost:5432/sqltest +#spring.datasource.driverClassName=org.postgresql.Driver +#spring.datasource.hikari.maximumPoolSize = 50 +queue.core.pack-processing-timeout=3000 +queue.rule-engine.pack-processing-timeout=3000 +queue.rule-engine.queues[0].name=Main +queue.rule-engine.queues[0].topic=tb_rule_engine.main +queue.rule-engine.queues[0].poll-interval=25 +queue.rule-engine.queues[0].partitions=3 +queue.rule-engine.queues[0].pack-processing-timeout=3000 +queue.rule-engine.queues[0].processing-strategy.type=SKIP_ALL_FAILURES +queue.rule-engine.queues[0].submit-strategy.type=BURST +sql.log_entity_queries=true diff --git a/dao/src/test/resources/sql/system-test-psql.sql b/dao/src/test/resources/sql/system-test-psql.sql new file mode 100644 index 0000000000..16dcb8c3ae --- /dev/null +++ b/dao/src/test/resources/sql/system-test-psql.sql @@ -0,0 +1,2 @@ +--PostgreSQL specific truncate to fit constraints +TRUNCATE TABLE device_credentials, device, device_profile, rule_node_state, rule_node, rule_chain; \ No newline at end of file diff --git a/pom.xml b/pom.xml index fa33548a56..2cc0f0d6b1 100755 --- a/pom.xml +++ b/pom.xml @@ -1637,6 +1637,17 @@ org.hsqldb hsqldb ${hsqldb.version} + + + org.testcontainers + postgresql + ${testcontainers.version} + test + + + org.testcontainers + jdbc + ${testcontainers.version} test