diff --git a/application/pom.xml b/application/pom.xml index f454a462b9..3378d52459 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -360,6 +360,7 @@ **/sql/*Test.java + **/psql/*Test.java **/nosql/*Test.java diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/id/EntityIdTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/id/EntityIdTest.java new file mode 100644 index 0000000000..a53124860c --- /dev/null +++ b/common/data/src/test/java/org/thingsboard/server/common/data/id/EntityIdTest.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import org.junit.Assert; +import org.junit.Test; + +public class EntityIdTest { + + @Test + public void givenConstantNullUuid_whenCompare_thenToStringEqualsPredefinedUuid() { + Assert.assertEquals("13814000-1dd2-11b2-8080-808080808080", EntityId.NULL_UUID.toString()); + } + +} \ No newline at end of file diff --git a/dao/pom.xml b/dao/pom.xml index 922b389bff..f490a0e8e5 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -201,6 +201,11 @@ org.springframework.boot spring-boot-starter-data-jpa + + org.springframework.boot + spring-boot-starter-test + test + org.springframework spring-test @@ -211,6 +216,16 @@ hsqldb test + + org.testcontainers + postgresql + test + + + org.testcontainers + jdbc + test + org.springframework spring-context-support @@ -239,7 +254,14 @@ maven-surefire-plugin ${surfire.version} + + **/sql/*Test.java + **/sql/*/*Test.java + **/psql/*Test.java + **/nosql/*Test.java + + **/*Test.java **/*TestSuite.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index 7b3ea36cfa..f567420947 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -223,6 +223,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { private static final String SELECT_API_USAGE_STATE = "(select aus.id, aus.created_time, aus.tenant_id, aus.entity_id, " + "coalesce((select title from tenant where id = aus.entity_id), (select title from customer where id = aus.entity_id)) as name " + "from api_usage_state as aus)"; + static final int MAX_LEVEL_DEFAULT = 50; //This value has to be reasonable small to prevent infinite recursion as early as possible static { entityTableMap.put(EntityType.ASSET, "asset"); @@ -239,18 +240,30 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { public static EntityType[] RELATION_QUERY_ENTITY_TYPES = new EntityType[]{ EntityType.TENANT, EntityType.CUSTOMER, EntityType.USER, EntityType.DASHBOARD, EntityType.ASSET, EntityType.DEVICE, EntityType.ENTITY_VIEW}; - private static final String HIERARCHICAL_QUERY_TEMPLATE = " FROM (WITH RECURSIVE related_entities(from_id, from_type, to_id, to_type, relation_type, lvl) AS (" + - " SELECT from_id, from_type, to_id, to_type, relation_type, 1 as lvl" + - " FROM relation" + + private static final String HIERARCHICAL_QUERY_TEMPLATE = " FROM (WITH RECURSIVE related_entities(from_id, from_type, to_id, to_type, lvl, path) AS (" + + " SELECT from_id, from_type, to_id, to_type," + + " 1 as lvl," + + " ARRAY[$in_id] as path" + // initial path + " FROM relation " + " WHERE $in_id = :relation_root_id and $in_type = :relation_root_type and relation_type_group = 'COMMON'" + + " GROUP BY from_id, from_type, to_id, to_type, lvl, path" + " UNION ALL" + - " SELECT r.from_id, r.from_type, r.to_id, r.to_type, r.relation_type, lvl + 1" + + " SELECT r.from_id, r.from_type, r.to_id, r.to_type," + + " (re.lvl + 1) as lvl, " + + " (re.path || ARRAY[r.$in_id]) as path" + " FROM relation r" + " INNER JOIN related_entities re ON" + " r.$in_id = re.$out_id and r.$in_type = re.$out_type and" + - " relation_type_group = 'COMMON' %s)" + - " SELECT re.$out_id entity_id, re.$out_type entity_type, max(re.lvl) lvl" + - " from related_entities re" + + " relation_type_group = 'COMMON' " + + " AND r.$in_id NOT IN (SELECT * FROM unnest(re.path)) " + + " %s" + + " GROUP BY r.from_id, r.from_type, r.to_id, r.to_type, (re.lvl + 1), (re.path || ARRAY[r.$in_id])" + + " )" + + " SELECT re.$out_id entity_id, re.$out_type entity_type, max(r_int.lvl) lvl" + + " from related_entities r_int" + + " INNER JOIN relation re ON re.from_id = r_int.from_id AND re.from_type = r_int.from_type" + + " AND re.to_id = r_int.to_id AND re.to_type = r_int.to_type" + + " AND re.relation_type_group = 'COMMON'" + " %s GROUP BY entity_id, entity_type) entity"; private static final String HIERARCHICAL_TO_QUERY_TEMPLATE = HIERARCHICAL_QUERY_TEMPLATE.replace("$in", "to").replace("$out", "from"); private static final String HIERARCHICAL_FROM_QUERY_TEMPLATE = HIERARCHICAL_QUERY_TEMPLATE.replace("$in", "from").replace("$out", "to"); @@ -580,7 +593,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { .append("nr.").append(fromOrTo).append("_type").append(" = re.").append(toOrFrom).append("_type"); notExistsPart.append(")"); - whereFilter += " and ( re.lvl = " + entityFilter.getMaxLevel() + " OR " + notExistsPart.toString() + ")"; + whereFilter += " and ( r_int.lvl = " + entityFilter.getMaxLevel() + " OR " + notExistsPart.toString() + ")"; } from = String.format(from, lvlFilter, whereFilter); String query = "( " + selectFields + from + ")"; @@ -659,7 +672,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { .append(whereFilter.toString().replaceAll("re\\.", "nr\\.")); notExistsPart.append(")"); - whereFilter.append(" and ( re.lvl = ").append(entityFilter.getMaxLevel()).append(" OR ").append(notExistsPart.toString()).append(")"); + whereFilter.append(" and ( r_int.lvl = ").append(entityFilter.getMaxLevel()).append(" OR ").append(notExistsPart.toString()).append(")"); } from = String.format(from, lvlFilter, " WHERE " + whereFilter); return "( " + selectFields + from + ")"; @@ -693,8 +706,12 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return whereFilter.toString(); } - private String getLvlFilter(int maxLevel) { - return maxLevel > 0 ? ("and lvl <= " + (maxLevel - 1)) : ""; + String getLvlFilter(int maxLevel) { + return "and re.lvl <= " + (getMaxLevel(maxLevel) - 1); + } + + int getMaxLevel(int maxLevel) { + return maxLevel > 0 ? maxLevel : MAX_LEVEL_DEFAULT; } private String getQueryTemplate(EntitySearchDirection direction) { diff --git a/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java new file mode 100644 index 0000000000..b89c70e8f4 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.junit.extensions.cpsuite.ClasspathSuite; +import org.junit.extensions.cpsuite.ClasspathSuite.ClassnameFilters; +import org.junit.runner.RunWith; + +@RunWith(ClasspathSuite.class) +@ClassnameFilters({ + "org.thingsboard.server.dao.service.psql.*SqlTest", + "org.thingsboard.server.dao.service.attributes.psql.*SqlTest", + "org.thingsboard.server.dao.service.event.psql.*SqlTest", + "org.thingsboard.server.dao.service.timeseries.psql.*SqlTest" +}) +public class PostgreSqlDaoServiceTestSuite { +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlInitializer.java b/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlInitializer.java new file mode 100644 index 0000000000..429fa1d711 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlInitializer.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import com.google.common.base.Charsets; +import com.google.common.io.Resources; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.net.URL; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; + +@Slf4j +public class PostgreSqlInitializer { + + private static final List sqlFiles = List.of( + "sql/schema-ts-psql.sql", + "sql/schema-entities.sql", + "sql/schema-entities-idx.sql", + "sql/system-data.sql", + "sql/system-test-psql.sql"); + private static final String dropAllTablesSqlFile = "sql/psql/drop-all-tables.sql"; + + public static void initDb(Connection conn) { + cleanUpDb(conn); + log.info("initialize Postgres DB..."); + try { + for (String sqlFile : sqlFiles) { + URL sqlFileUrl = Resources.getResource(sqlFile); + String sql = Resources.toString(sqlFileUrl, Charsets.UTF_8); + conn.createStatement().execute(sql); + } + } catch (IOException | SQLException e) { + throw new RuntimeException("Unable to init the Postgres database. Reason: " + e.getMessage(), e); + } + log.info("Postgres DB is initialized!"); + } + + private static void cleanUpDb(Connection conn) { + log.info("clean up Postgres DB..."); + try { + URL dropAllTableSqlFileUrl = Resources.getResource(dropAllTablesSqlFile); + String dropAllTablesSql = Resources.toString(dropAllTableSqlFileUrl, Charsets.UTF_8); + conn.createStatement().execute(dropAllTablesSql); + } catch (IOException | SQLException e) { + throw new RuntimeException("Unable to clean up the Postgres database. Reason: " + e.getMessage(), e); + } + } +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java index 2a1ff95062..cfd7923969 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java @@ -47,7 +47,7 @@ import java.util.stream.Collectors; import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; -public class BaseDeviceProfileServiceTest extends AbstractServiceTest { +public abstract class BaseDeviceProfileServiceTest extends AbstractServiceTest { private IdComparator idComparator = new IdComparator<>(); private IdComparator deviceProfileInfoIdComparator = new IdComparator<>(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index 831d92965b..e58191b60d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -17,14 +17,17 @@ package org.thingsboard.server.dao.service; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; +import org.hamcrest.Matchers; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.ResultSetExtractor; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; @@ -69,6 +72,7 @@ import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; +import org.thingsboard.server.dao.sql.relation.RelationRepository; import org.thingsboard.server.dao.timeseries.TimeseriesService; import java.util.ArrayList; @@ -82,9 +86,13 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.Assert.assertEquals; +import static org.hamcrest.MatcherAssert.assertThat; +@Slf4j public abstract class BaseEntityServiceTest extends AbstractServiceTest { + static final int ENTITY_COUNT = 5; + @Autowired private AttributesService attributesService; @@ -96,6 +104,9 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { @Autowired private JdbcTemplate template; + @Autowired + private RelationRepository relationRepository; + @Before public void before() { Tenant tenant = new Tenant(); @@ -110,7 +121,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { tenantService.deleteTenant(tenantId); } - + @Test public void testCountEntitiesByQuery() throws InterruptedException { List devices = new ArrayList<>(); @@ -154,12 +165,12 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { Assert.assertEquals(0, count); } - + @Test public void testCountHierarchicalEntitiesByQuery() throws InterruptedException { List assets = new ArrayList<>(); List devices = new ArrayList<>(); - createTestHierarchy(assets, devices, new ArrayList<>(), new ArrayList<>(), new ArrayList<>(), new ArrayList<>()); + createTestHierarchy(tenantId, assets, devices, new ArrayList<>(), new ArrayList<>(), new ArrayList<>(), new ArrayList<>()); RelationsQueryFilter filter = new RelationsQueryFilter(); filter.setRootEntity(tenantId); @@ -168,7 +179,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { EntityCountQuery countQuery = new EntityCountQuery(filter); long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(30, count); + Assert.assertEquals(31, count); //due to the loop relations in hierarchy, the TenantId included in total count (1*Tenant + 5*Asset + 5*5*Devices = 31) filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("Contains", Collections.singletonList(EntityType.DEVICE)))); count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); @@ -304,11 +315,25 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { @Test public void testHierarchicalFindEntityDataWithAttributesByQuery() throws ExecutionException, InterruptedException { + doTestHierarchicalFindEntityDataWithAttributesByQuery(0, false); + } + + @Test + public void testHierarchicalFindEntityDataWithAttributesByQueryWithLevel() throws ExecutionException, InterruptedException { + doTestHierarchicalFindEntityDataWithAttributesByQuery(2, false); + } + + @Test + public void testHierarchicalFindEntityDataWithAttributesByQueryWithLastLevelOnly() throws ExecutionException, InterruptedException { + doTestHierarchicalFindEntityDataWithAttributesByQuery(2, true); + } + + private void doTestHierarchicalFindEntityDataWithAttributesByQuery(final int maxLevel, final boolean fetchLastLevelOnly) throws ExecutionException, InterruptedException { List assets = new ArrayList<>(); List devices = new ArrayList<>(); List temperatures = new ArrayList<>(); List highTemperatures = new ArrayList<>(); - createTestHierarchy(assets, devices, new ArrayList<>(), new ArrayList<>(), temperatures, highTemperatures); + createTestHierarchy(tenantId, assets, devices, new ArrayList<>(), new ArrayList<>(), temperatures, highTemperatures); List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { @@ -321,6 +346,8 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { filter.setRootEntity(tenantId); filter.setDirection(EntitySearchDirection.FROM); filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("Contains", Collections.singletonList(EntityType.DEVICE)))); + filter.setMaxLevel(maxLevel); + filter.setFetchLastLevelOnly(fetchLastLevelOnly); EntityDataSortOrder sortOrder = new EntityDataSortOrder( new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC @@ -373,14 +400,13 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { deviceService.deleteDevicesByTenantId(tenantId); } - @Test public void testHierarchicalFindDevicesWithAttributesByQuery() throws ExecutionException, InterruptedException { List assets = new ArrayList<>(); List devices = new ArrayList<>(); List temperatures = new ArrayList<>(); List highTemperatures = new ArrayList<>(); - createTestHierarchy(assets, devices, new ArrayList<>(), new ArrayList<>(), temperatures, highTemperatures); + createTestHierarchy(tenantId, assets, devices, new ArrayList<>(), new ArrayList<>(), temperatures, highTemperatures); List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { @@ -393,6 +419,8 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { filter.setRootEntity(tenantId); filter.setDirection(EntitySearchDirection.FROM); filter.setRelationType("Contains"); + filter.setMaxLevel(2); + filter.setFetchLastLevelOnly(true); EntityDataSortOrder sortOrder = new EntityDataSortOrder( new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC @@ -446,14 +474,14 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { deviceService.deleteDevicesByTenantId(tenantId); } - + @Test public void testHierarchicalFindAssetsWithAttributesByQuery() throws ExecutionException, InterruptedException { List assets = new ArrayList<>(); List devices = new ArrayList<>(); List consumptions = new ArrayList<>(); List highConsumptions = new ArrayList<>(); - createTestHierarchy(assets, devices, consumptions, highConsumptions, new ArrayList<>(), new ArrayList<>()); + createTestHierarchy(tenantId, assets, devices, consumptions, highConsumptions, new ArrayList<>(), new ArrayList<>()); List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < assets.size(); i++) { @@ -518,8 +546,8 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { deviceService.deleteDevicesByTenantId(tenantId); } - private void createTestHierarchy(List assets, List devices, List consumptions, List highConsumptions, List temperatures, List highTemperatures) throws InterruptedException { - for (int i = 0; i < 5; i++) { + private void createTestHierarchy(TenantId tenantId, List assets, List devices, List consumptions, List highConsumptions, List temperatures, List highTemperatures) throws InterruptedException { + for (int i = 0; i < ENTITY_COUNT; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); asset.setName("Asset" + i); @@ -529,18 +557,19 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { //TO make sure devices have different created time Thread.sleep(1); assets.add(asset); - EntityRelation er = new EntityRelation(); - er.setFrom(tenantId); - er.setTo(asset.getId()); - er.setType("Manages"); - er.setTypeGroup(RelationTypeGroup.COMMON); - relationService.saveRelation(tenantId, er); + createRelation(tenantId, "Manages", tenantId, asset.getId()); long consumption = (long) (Math.random() * 100); consumptions.add(consumption); if (consumption > 50) { highConsumptions.add(consumption); } - for (int j = 0; j < 5; j++) { + + //tenant -> asset : one-to-one but many edges + for (int n = 0; n < ENTITY_COUNT; n++) { + createRelation(tenantId, "UseCase-" + n, tenantId, asset.getId()); + } + + for (int j = 0; j < ENTITY_COUNT; j++) { Device device = new Device(); device.setTenantId(tenantId); device.setName("A" + i + "Device" + j); @@ -550,22 +579,125 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { //TO make sure devices have different created time Thread.sleep(1); devices.add(device); - er = new EntityRelation(); - er.setFrom(asset.getId()); - er.setTo(device.getId()); - er.setType("Contains"); - er.setTypeGroup(RelationTypeGroup.COMMON); - relationService.saveRelation(tenantId, er); + createRelation(tenantId, "Contains", asset.getId(), device.getId()); long temperature = (long) (Math.random() * 100); temperatures.add(temperature); if (temperature > 45) { highTemperatures.add(temperature); } + + //asset -> device : one-to-one but many edges + for (int n = 0; n < ENTITY_COUNT; n++) { + createRelation(tenantId, "UseCase-" + n, asset.getId(), device.getId()); + } + } + } + + //asset -> device one-to-many shared with other assets + for (int n = 0; n < devices.size(); n = n + ENTITY_COUNT) { + createRelation(tenantId, "SharedWithAsset0", assets.get(0).getId(), devices.get(n).getId()); + } + + createManyCustomRelationsBetweenTwoNodes(tenantId, "UseCase", assets, devices); + createHorizontalRingRelations(tenantId, "Ring(Loop)-Ast", assets); + createLoopRelations(tenantId, "Loop-Tnt-Ast-Dev", tenantId, assets.get(0).getId(), devices.get(0).getId()); + createLoopRelations(tenantId, "Loop-Tnt-Ast", tenantId, assets.get(1).getId()); + createLoopRelations(tenantId, "Loop-Ast-Tnt-Ast", assets.get(2).getId(), tenantId, assets.get(3).getId()); + + //printAllRelations(); + } + + private ResultSetExtractor>> getListResultSetExtractor() { + return rs -> { + List> list = new ArrayList<>(); + final int columnCount = rs.getMetaData().getColumnCount(); + List columns = new ArrayList<>(columnCount); + for (int i = 1; i <= columnCount; i++) { + columns.add(rs.getMetaData().getColumnName(i)); + } + list.add(columns); + while (rs.next()) { + List data = new ArrayList<>(columnCount); + for (int i = 1; i <= columnCount; i++) { + data.add(rs.getString(i)); + } + list.add(data); } + return list; + }; + } + + /* + * This useful to reproduce exact data in the PostgreSQL and play around with pgadmin query and analyze tool + * */ + private void printAllRelations() { + System.out.println("" + + "DO\n" + + "$$\n" + + " DECLARE\n" + + " someint integer;\n" + + " BEGIN\n" + + " DROP TABLE IF EXISTS relation_test;\n" + + " CREATE TABLE IF NOT EXISTS relation_test\n" + + " (\n" + + " from_id uuid,\n" + + " from_type varchar(255),\n" + + " to_id uuid,\n" + + " to_type varchar(255),\n" + + " relation_type_group varchar(255),\n" + + " relation_type varchar(255),\n" + + " additional_info varchar,\n" + + " CONSTRAINT relation_test_pkey PRIMARY KEY (from_id, from_type, relation_type_group, relation_type, to_id, to_type)\n" + + " );"); + + relationRepository.findAll().forEach(r -> + System.out.printf("INSERT INTO relation_test (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info)" + + " VALUES (%s, %s, %s, %s, %s, %s, %s);\n", + quote(r.getFromId()), quote(r.getFromType()), quote(r.getToId()), quote(r.getToType()), + quote(r.getRelationTypeGroup()), quote(r.getRelationType()), quote(r.getAdditionalInfo())) + ); + + System.out.println("" + + " END\n" + + "$$;"); + } + + private String quote(Object s) { + return s == null ? null : "'" + s + "'"; + } + + void createLoopRelations(TenantId tenantId, String type, EntityId... ids) { + assertThat("ids lenght", ids.length, Matchers.greaterThanOrEqualTo(1)); + //chain all from the head to the tail + for (int i = 1; i < ids.length; i++) { + relationService.saveRelation(tenantId, new EntityRelation(ids[i - 1], ids[i], type, RelationTypeGroup.COMMON)); + } + //chain tail -> head + relationService.saveRelation(tenantId, new EntityRelation(ids[ids.length - 1], ids[0], type, RelationTypeGroup.COMMON)); + } + + void createHorizontalRingRelations(TenantId tenantId, String type, List assets) { + createLoopRelations(tenantId, type, assets.stream().map(Asset::getId).toArray(EntityId[]::new)); + } + + void createManyCustomRelationsBetweenTwoNodes(TenantId tenantId, String type, List assets, List devices) { + for (int i = 1; i <= 5; i++) { + final String typeI = type + i; + createOneToManyRelations(tenantId, typeI, tenantId, assets.stream().map(Asset::getId).collect(Collectors.toList())); + assets.forEach(asset -> + createOneToManyRelations(tenantId, typeI, asset.getId(), devices.stream().map(Device::getId).collect(Collectors.toList()))); } } - + void createOneToManyRelations(TenantId tenantId, String type, EntityId from, List toIds) { + toIds.forEach(toId -> createRelation(tenantId, type, from, toId)); + } + + void createRelation(TenantId tenantId, String type, EntityId from, EntityId toId) { + relationService.saveRelation(tenantId, new EntityRelation(from, toId, type, RelationTypeGroup.COMMON)); + } + + @Test public void testSimpleFindEntityDataByQuery() throws InterruptedException { List devices = new ArrayList<>(); @@ -871,7 +1003,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } @Test - public void testBuildNumericPredicateQueryOperations() throws ExecutionException, InterruptedException{ + public void testBuildNumericPredicateQueryOperations() throws ExecutionException, InterruptedException { List devices = new ArrayList<>(); List temperatures = new ArrayList<>(); @@ -1031,7 +1163,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { deviceService.deleteDevicesByTenantId(tenantId); } - + @Test public void testFindEntityDataByQueryWithTimeseries() throws ExecutionException, InterruptedException { @@ -1122,7 +1254,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } @Test - public void testBuildStringPredicateQueryOperations() throws ExecutionException, InterruptedException{ + public void testBuildStringPredicateQueryOperations() throws ExecutionException, InterruptedException { List devices = new ArrayList<>(); List attributeStrings = new ArrayList<>(); @@ -1142,11 +1274,11 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { devices.add(deviceService.saveDevice(device)); //TO make sure devices have different created time Thread.sleep(1); - List operationValues= Arrays.asList(StringFilterPredicate.StringOperation.values()); + List operationValues = Arrays.asList(StringFilterPredicate.StringOperation.values()); StringFilterPredicate.StringOperation operation = operationValues.get(new Random().nextInt(operationValues.size())); String operationName = operation.name(); attributeStrings.add(operationName); - switch(operation){ + switch (operation) { case EQUAL: equalStrings.add(operationName); notContainsStrings.add(operationName); @@ -1302,7 +1434,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } @Test - public void testBuildStringPredicateQueryOperationsForEntityType() throws ExecutionException, InterruptedException{ + public void testBuildStringPredicateQueryOperationsForEntityType() throws ExecutionException, InterruptedException { List devices = new ArrayList<>(); @@ -1419,7 +1551,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } @Test - public void testBuildSimplePredicateQueryOperations() throws InterruptedException{ + public void testBuildSimplePredicateQueryOperations() throws InterruptedException { List devices = new ArrayList<>(); @@ -1492,7 +1624,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { return loadedEntities; } - private List createStringKeyFilters(String key, EntityKeyType keyType, StringFilterPredicate.StringOperation operation, String value){ + private List createStringKeyFilters(String key, EntityKeyType keyType, StringFilterPredicate.StringOperation operation, String value) { KeyFilter filter = new KeyFilter(); filter.setKey(new EntityKey(keyType, key)); StringFilterPredicate predicate = new StringFilterPredicate(); @@ -1503,7 +1635,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { return Collections.singletonList(filter); } - private KeyFilter createNumericKeyFilter(String key, EntityKeyType keyType, NumericFilterPredicate.NumericOperation operation, double value){ + private KeyFilter createNumericKeyFilter(String key, EntityKeyType keyType, NumericFilterPredicate.NumericOperation operation, double value) { KeyFilter filter = new KeyFilter(); filter.setKey(new EntityKey(keyType, key)); NumericFilterPredicate predicate = new NumericFilterPredicate(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOAuth2ConfigTemplateServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOAuth2ConfigTemplateServiceTest.java index c3073434e0..743591863a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOAuth2ConfigTemplateServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOAuth2ConfigTemplateServiceTest.java @@ -31,7 +31,7 @@ import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService; import java.util.Arrays; import java.util.UUID; -public class BaseOAuth2ConfigTemplateServiceTest extends AbstractServiceTest { +public abstract class BaseOAuth2ConfigTemplateServiceTest extends AbstractServiceTest { @Autowired protected OAuth2ConfigTemplateService oAuth2ConfigTemplateService; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOAuth2ServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOAuth2ServiceTest.java index 4077bb2176..0ad3470c99 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOAuth2ServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOAuth2ServiceTest.java @@ -43,7 +43,7 @@ import java.util.List; import java.util.UUID; import java.util.stream.Collectors; -public class BaseOAuth2ServiceTest extends AbstractServiceTest { +public abstract class BaseOAuth2ServiceTest extends AbstractServiceTest { private static final OAuth2Info EMPTY_PARAMS = new OAuth2Info(false, Collections.emptyList()); @Autowired diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java index 2ebb955c61..d77143f0a0 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java @@ -34,7 +34,7 @@ import java.util.Collections; import java.util.List; import java.util.stream.Collectors; -public class BaseTenantProfileServiceTest extends AbstractServiceTest { +public abstract class BaseTenantProfileServiceTest extends AbstractServiceTest { private IdComparator idComparator = new IdComparator<>(); private IdComparator tenantProfileInfoIdComparator = new IdComparator<>(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DaoPostgreSqlTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DaoPostgreSqlTest.java new file mode 100644 index 0000000000..43ed1f4d02 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DaoPostgreSqlTest.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service; + +import org.springframework.test.context.TestPropertySource; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Inherited +@Documented +@TestPropertySource(locations = {"classpath:application-test.properties", "classpath:psql-test.properties"}) +public @interface DaoPostgreSqlTest { +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/sql/EntityServiceSqlTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/psql/EntityServicePostgreSqlTest.java similarity index 77% rename from dao/src/test/java/org/thingsboard/server/dao/service/sql/EntityServiceSqlTest.java rename to dao/src/test/java/org/thingsboard/server/dao/service/psql/EntityServicePostgreSqlTest.java index 0c59ae49c8..5b8c6d4aca 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/sql/EntityServiceSqlTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/psql/EntityServicePostgreSqlTest.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.service.sql; +package org.thingsboard.server.dao.service.psql; import org.thingsboard.server.dao.service.BaseEntityServiceTest; -import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.service.DaoPostgreSqlTest; -@DaoSqlTest -public class EntityServiceSqlTest extends BaseEntityServiceTest { +@DaoPostgreSqlTest +public class EntityServicePostgreSqlTest extends BaseEntityServiceTest { } diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepositoryTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepositoryTest.java new file mode 100644 index 0000000000..a0c716d51f --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepositoryTest.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.query; + +import org.junit.Test; +import org.thingsboard.server.common.data.id.CustomerId; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.BDDMockito.willCallRealMethod; +import static org.mockito.Mockito.mock; + +public class DefaultEntityQueryRepositoryTest { + + /* + * This value has to be reasonable small to prevent infinite recursion as early as possible + * */ + @Test + public void givenDefaultMaxLevel_whenStaticConstant_thenEqualsTo() { + assertThat(DefaultEntityQueryRepository.MAX_LEVEL_DEFAULT, equalTo(10)); + } + + @Test + public void givenMaxLevelZeroOrNegative_whenGetMaxLevel_thenReturnDefaultMaxLevel() { + DefaultEntityQueryRepository repo = mock(DefaultEntityQueryRepository.class); + willCallRealMethod().given(repo).getMaxLevel(anyInt()); + assertThat(repo.getMaxLevel(0), equalTo(DefaultEntityQueryRepository.MAX_LEVEL_DEFAULT)); + assertThat(repo.getMaxLevel(-1), equalTo(DefaultEntityQueryRepository.MAX_LEVEL_DEFAULT)); + assertThat(repo.getMaxLevel(-2), equalTo(DefaultEntityQueryRepository.MAX_LEVEL_DEFAULT)); + assertThat(repo.getMaxLevel(Integer.MIN_VALUE), equalTo(DefaultEntityQueryRepository.MAX_LEVEL_DEFAULT)); + } + + @Test + public void givenMaxLevelPositive_whenGetMaxLevel_thenValueTheSame() { + DefaultEntityQueryRepository repo = mock(DefaultEntityQueryRepository.class); + willCallRealMethod().given(repo).getMaxLevel(anyInt()); + assertThat(repo.getMaxLevel(1), equalTo(1)); + assertThat(repo.getMaxLevel(2), equalTo(2)); + assertThat(repo.getMaxLevel(Integer.MAX_VALUE), equalTo(Integer.MAX_VALUE)); + } + +} diff --git a/dao/src/test/resources/psql-test.properties b/dao/src/test/resources/psql-test.properties new file mode 100644 index 0000000000..fb65966acf --- /dev/null +++ b/dao/src/test/resources/psql-test.properties @@ -0,0 +1,47 @@ +database.ts.type=sql +database.ts_latest.type=sql +sql.ts_inserts_executor_type=fixed +sql.ts_inserts_fixed_thread_pool_size=200 +sql.ts_key_value_partitioning=MONTHS +# +spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true +spring.jpa.properties.hibernate.order_by.default_null_ordering=last +spring.jpa.properties.hibernate.jdbc.log.warnings=false +spring.jpa.show-sql=false +spring.jpa.hibernate.ddl-auto=none +spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect +spring.datasource.username=postgres +spring.datasource.password=postgres +spring.datasource.url=jdbc:tc:postgresql:12.8:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw&?TC_INITFUNCTION=org.thingsboard.server.dao.PostgreSqlInitializer::initDb +spring.datasource.driverClassName=org.testcontainers.jdbc.ContainerDatabaseDriver +#org.postgresql.Driver +spring.datasource.hikari.maximumPoolSize=50 +service.type=monolith +#database.ts.type=timescale +#database.ts.type=sql +#database.entities.type=sql +# +#sql.ts_inserts_executor_type=fixed +#sql.ts_inserts_fixed_thread_pool_size=200 +#sql.ts_key_value_partitioning=MONTHS +# +#spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true +#spring.jpa.show-sql=false +#spring.jpa.hibernate.ddl-auto=none +#spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect +# +#spring.datasource.username=postgres +#spring.datasource.password=postgres +#spring.datasource.url=jdbc:postgresql://localhost:5432/sqltest +#spring.datasource.driverClassName=org.postgresql.Driver +#spring.datasource.hikari.maximumPoolSize = 50 +queue.core.pack-processing-timeout=3000 +queue.rule-engine.pack-processing-timeout=3000 +queue.rule-engine.queues[0].name=Main +queue.rule-engine.queues[0].topic=tb_rule_engine.main +queue.rule-engine.queues[0].poll-interval=25 +queue.rule-engine.queues[0].partitions=3 +queue.rule-engine.queues[0].pack-processing-timeout=3000 +queue.rule-engine.queues[0].processing-strategy.type=SKIP_ALL_FAILURES +queue.rule-engine.queues[0].submit-strategy.type=BURST +sql.log_entity_queries=true diff --git a/dao/src/test/resources/sql/system-test-psql.sql b/dao/src/test/resources/sql/system-test-psql.sql new file mode 100644 index 0000000000..16dcb8c3ae --- /dev/null +++ b/dao/src/test/resources/sql/system-test-psql.sql @@ -0,0 +1,2 @@ +--PostgreSQL specific truncate to fit constraints +TRUNCATE TABLE device_credentials, device, device_profile, rule_node_state, rule_node, rule_chain; \ No newline at end of file diff --git a/pom.xml b/pom.xml index b2ce88e87d..0ca42294fa 100755 --- a/pom.xml +++ b/pom.xml @@ -1643,6 +1643,17 @@ org.hsqldb hsqldb ${hsqldb.version} + + + org.testcontainers + postgresql + ${testcontainers.version} + test + + + org.testcontainers + jdbc + ${testcontainers.version} test