From 2b1a43792cb8e2e676ade4fd14125ffe7a28a26e Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 15 Jun 2020 18:25:19 +0300 Subject: [PATCH] Relations Query Initial Implementation --- .../query/DefaultEntityQueryRepository.java | 195 +++++++++++++++--- .../server/dao/SqlDaoServiceTestSuite.java | 2 +- .../dao/service/BaseEntityServiceTest.java | 71 ++++++- 3 files changed, 230 insertions(+), 38 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index c057ce477c..8b9c6d3a97 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Repository; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AssetTypeFilter; @@ -31,17 +32,19 @@ import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityFilter; -import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityFilterType; import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.query.EntityNameFilter; import org.thingsboard.server.common.data.query.EntityViewTypeFilter; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; import org.thingsboard.server.common.data.query.SingleEntityFilter; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.EntityTypeFilter; import org.thingsboard.server.dao.util.SqlDao; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; import java.math.BigInteger; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -56,6 +59,7 @@ import java.util.stream.Collectors; public class DefaultEntityQueryRepository implements EntityQueryRepository { private static final Map entityTableMap = new HashMap<>(); + static { entityTableMap.put(EntityType.ASSET, "asset"); entityTableMap.put(EntityType.DEVICE, "device"); @@ -73,9 +77,9 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query) { EntityType entityType = resolveEntityType(query.getEntityFilter()); String countQuery = String.format("select count(e.id) from %s e where %s", - entityTableMap.get(entityType), this.buildEntityWhere(tenantId, customerId, query.getEntityFilter(), + getEntityTableQuery(query.getEntityFilter(), entityType), this.buildEntityWhere(tenantId, customerId, query.getEntityFilter(), Collections.emptyList(), entityType)); - return ((BigInteger)entityManager.createNativeQuery(countQuery) + return ((BigInteger) entityManager.createNativeQuery(countQuery) .getSingleResult()).longValue(); } @@ -122,12 +126,12 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { String fromClause = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result", topSelection, entityFieldsSelection, - entityTableMap.get(entityType), + getEntityTableQuery(query.getEntityFilter(), entityType), entityWhereClause, latestJoins, whereClause); - int totalElements = ((BigInteger)entityManager.createNativeQuery(String.format("select count(*) %s", fromClause)) + int totalElements = ((BigInteger) entityManager.createNativeQuery(String.format("select count(*) %s", fromClause)) .getSingleResult()).intValue(); String dataQuery = String.format("select * %s", fromClause); @@ -154,26 +158,34 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } private String buildEntityWhere(TenantId tenantId, - CustomerId customerId, - EntityFilter entityFilter, - List entityFieldsFilters, - EntityType entityType) { - String permissionQuery = this.buildPermissionQuery(tenantId, customerId, entityType); + CustomerId customerId, + EntityFilter entityFilter, + List entityFieldsFilters, + EntityType entityType) { + String permissionQuery = this.buildPermissionQuery(entityFilter, tenantId, customerId, entityType); String entityFilterQuery = this.buildEntityFilterQuery(entityFilter); + String result = permissionQuery; + if (!entityFilterQuery.isEmpty()) { + result += " and " + entityFilterQuery; + } if (!entityFieldsFilters.isEmpty()) { - String entityFieldsQuery = EntityKeyMapping.buildQuery(entityFieldsFilters); - return String.join(" and ", permissionQuery, entityFilterQuery, entityFieldsQuery); - } else { - return String.join(" and ", permissionQuery, entityFilterQuery); + result += " and " + entityFieldsFilters; } + return result; } - private String buildPermissionQuery(TenantId tenantId, CustomerId customerId, EntityType entityType) { - String permissionQuery = String.format("e.tenant_id='%s'", UUIDConverter.fromTimeUUID(tenantId.getId())); - if (entityType != EntityType.TENANT && entityType != EntityType.CUSTOMER) { - permissionQuery = String.format("%s and e.customer_id='%s'", permissionQuery, UUIDConverter.fromTimeUUID(customerId.getId())); + private String buildPermissionQuery(EntityFilter entityFilter, TenantId tenantId, CustomerId customerId, EntityType entityType) { + if (entityFilter.getType().equals(EntityFilterType.RELATIONS_QUERY)) { + return String.format("e.tenant_id='%s' and e.customer_id='%s'", UUIDConverter.fromTimeUUID(tenantId.getId()), UUIDConverter.fromTimeUUID(customerId.getId())); + } else { + if (entityType == EntityType.TENANT) { + return String.format("e.id='%s'", UUIDConverter.fromTimeUUID(tenantId.getId())); + } else if (entityType == EntityType.CUSTOMER) { + return String.format("e.tenant_id='%s' and e.id='%s'", UUIDConverter.fromTimeUUID(tenantId.getId()), UUIDConverter.fromTimeUUID(customerId.getId())); + } else { + return String.format("e.tenant_id='%s' and e.customer_id='%s'", UUIDConverter.fromTimeUUID(tenantId.getId()), UUIDConverter.fromTimeUUID(customerId.getId())); + } } - return permissionQuery; } private String buildEntityFilterQuery(EntityFilter entityFilter) { @@ -188,12 +200,129 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { case DEVICE_TYPE: case ENTITY_VIEW_TYPE: return this.typeQuery(entityFilter); + case RELATIONS_QUERY: + return ""; default: throw new RuntimeException("Not implemented!"); } } - private String buildWhere(List selectionMapping, List latestFiltersMapping, String searchText) { + private String getEntityTableQuery(EntityFilter entityFilter, EntityType entityType) { + switch (entityFilter.getType()) { + case RELATIONS_QUERY: + return relationQuery((RelationsQueryFilter) entityFilter); + default: + return entityTableMap.get(entityType); + } + } + + private String relationQuery(RelationsQueryFilter entityFilter) { + EntityId rootId = entityFilter.getRootEntity(); + String lvlFilter = entityFilter.getMaxLevel() > 0 ? ("and lvl <= " + (entityFilter.getMaxLevel() - 1)) : ""; + //TODO: refactoring + String selectFields = " select CASE\n" + + " WHEN entity.entity_type = 'DEVICE'\n" + + " THEN (select tenant_id from device where id = entity_id)\n" + + " WHEN entity.entity_type = 'ASSET'\n" + + " THEN (select tenant_id from asset where id = entity_id)\n" + + " WHEN entity.entity_type = 'CUSTOMER'\n" + + " THEN (select tenant_id from customer where id = entity_id)\n" + + " WHEN entity.entity_type = 'TENANT'\n" + + " THEN entity_id\n" + + " END as tenant_id,\n" + + " CASE\n" + + " WHEN entity.entity_type = 'DEVICE'\n" + + " THEN (select customer_id from device where id = entity_id)\n" + + " WHEN entity.entity_type = 'ASSET'\n" + + " THEN (select customer_id from asset where id = entity_id)\n" + + " WHEN entity.entity_type = 'CUSTOMER'\n" + + " THEN entity_id\n" + + " WHEN entity.entity_type = 'TENANT'\n" + + " THEN '1b21dd2138140008080808080808080'\n" + + " END as customer_id,\n" + + " entity.entity_id as id,\n" + + " CASE\n" + + " WHEN entity.entity_type = 'DEVICE'\n" + + " THEN (select type from device where id = entity_id)\n" + + " WHEN entity.entity_type = 'ASSET' THEN (select type from asset where id = entity_id)\n" + + " ELSE entity.entity_type\n" + + " END as type,\n" + + " CASE\n" + + " WHEN entity.entity_type = 'DEVICE'\n" + + " THEN (select name from device where id = entity_id)\n" + + " WHEN entity.entity_type = 'ASSET' THEN (select name from asset where id = entity_id)\n" + + " WHEN entity.entity_type = 'CUSTOMER'\n" + + " THEN (select title from customer where id = entity_id)\n" + + " WHEN entity.entity_type = 'TENANT'\n" + + " THEN (select title from tenant where id = entity_id)\n" + + " ELSE entity.entity_type\n" + + " END as name,\n" + + " CASE\n" + + " WHEN entity.entity_type = 'DEVICE'\n" + + " THEN (select label from device where id = entity_id)\n" + + " WHEN entity.entity_type = 'ASSET' THEN (select label from asset where id = entity_id)\n" + + " WHEN entity.entity_type = 'CUSTOMER'\n" + + " THEN (select title from customer where id = entity_id)\n" + + " WHEN entity.entity_type = 'TENANT'\n" + + " THEN (select title from tenant where id = entity_id)\n" + + " ELSE entity.entity_type\n" + + " END as label,\n" + + " entity.entity_type as entity_type"; + + String fromTemplate = " FROM (WITH RECURSIVE related_entities(from_id, from_type, to_id, to_type, relation_type, lvl) AS (" + + " SELECT from_id, from_type, to_id, to_type, relation_type, 1 as lvl" + + " FROM relation" + + " WHERE $in_id = '%s' and $in_type = '%s' and relation_type_group = 'COMMON'" + + " UNION ALL" + + " SELECT r.from_id, r.from_type, r.to_id, r.to_type, r.relation_type, lvl + 1" + + " FROM relation r" + + " INNER JOIN related_entities re ON" + + " r.$in_id = re.$out_id and r.$in_type = re.$out_type and" + + " relation_type_group = 'COMMON' %s)" + + " SELECT re.$out_id entity_id, re.$out_type entity_type, re.lvl lvl" + + " from related_entities re" + + " %s ) entity"; + + String from; + if (entityFilter.getDirection().equals(EntitySearchDirection.FROM)) { + from = fromTemplate.replace("$in", "from").replace("$out", "to"); + } else { + from = fromTemplate.replace("$in", "to").replace("$out", "from"); + } + + StringBuilder whereFilter; + if (entityFilter.getFilters() != null && !entityFilter.getFilters().isEmpty()) { + whereFilter = new StringBuilder(" WHERE "); + boolean first = true; + boolean single = entityFilter.getFilters().size() == 1; + for (EntityTypeFilter etf : entityFilter.getFilters()) { + if (first) { + first = false; + } else { + whereFilter.append(" AND "); + } + String relationType = etf.getRelationType(); + String types = etf.getEntityTypes().stream().map(type -> "'" + type + "'").collect(Collectors.joining(", ")); + if (!single) { + whereFilter.append(" ("); + } + whereFilter.append(" re.relation_type = '").append(relationType).append("' and re.") + .append(entityFilter.getDirection().equals(EntitySearchDirection.FROM) ? "to" : "from") + .append("_type in (").append(types).append(")"); + if (!single) { + whereFilter.append(" )"); + } + } + } else { + whereFilter = new StringBuilder(); + } + from = String.format(from, UUIDConverter.fromTimeUUID(rootId.getId()), rootId.getEntityType().name(), lvlFilter, whereFilter); + return "( " + selectFields + from + ")"; + } + + + private String buildWhere + (List selectionMapping, List latestFiltersMapping, String searchText) { String latestFilters = EntityKeyMapping.buildQuery(latestFiltersMapping); String textSearchQuery = this.buildTextSearchQuery(selectionMapping, searchText); String query; @@ -241,16 +370,16 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { String name; switch (filter.getType()) { case ASSET_TYPE: - type = ((AssetTypeFilter)filter).getAssetType(); - name = ((AssetTypeFilter)filter).getAssetNameFilter(); + type = ((AssetTypeFilter) filter).getAssetType(); + name = ((AssetTypeFilter) filter).getAssetNameFilter(); break; case DEVICE_TYPE: - type = ((DeviceTypeFilter)filter).getDeviceType(); - name = ((DeviceTypeFilter)filter).getDeviceNameFilter(); + type = ((DeviceTypeFilter) filter).getDeviceType(); + name = ((DeviceTypeFilter) filter).getDeviceNameFilter(); break; case ENTITY_VIEW_TYPE: - type = ((EntityViewTypeFilter)filter).getEntityViewType(); - name = ((EntityViewTypeFilter)filter).getEntityViewNameFilter(); + type = ((EntityViewTypeFilter) filter).getEntityViewType(); + name = ((EntityViewTypeFilter) filter).getEntityViewNameFilter(); break; default: throw new RuntimeException("Not supported!"); @@ -261,11 +390,11 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { private EntityType resolveEntityType(EntityFilter entityFilter) { switch (entityFilter.getType()) { case SINGLE_ENTITY: - return ((SingleEntityFilter)entityFilter).getSingleEntity().getEntityType(); + return ((SingleEntityFilter) entityFilter).getSingleEntity().getEntityType(); case ENTITY_LIST: - return ((EntityListFilter)entityFilter).getEntityType(); + return ((EntityListFilter) entityFilter).getEntityType(); case ENTITY_NAME: - return ((EntityNameFilter)entityFilter).getEntityType(); + return ((EntityNameFilter) entityFilter).getEntityType(); case ASSET_TYPE: case ASSET_SEARCH_QUERY: return EntityType.ASSET; @@ -275,8 +404,10 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { case ENTITY_VIEW_TYPE: case ENTITY_VIEW_SEARCH_QUERY: return EntityType.ENTITY_VIEW; - default: - throw new RuntimeException("Not implemented!"); + case RELATIONS_QUERY: + return ((RelationsQueryFilter) entityFilter).getRootEntity().getEntityType(); + default: + throw new RuntimeException("Not implemented!"); } } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java index 32fd45c188..f315075085 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java +++ b/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java @@ -24,7 +24,7 @@ import java.util.Arrays; @RunWith(ClasspathSuite.class) @ClassnameFilters({ - "org.thingsboard.server.dao.service.*ServiceSqlTest" + "org.thingsboard.server.dao.service.sql.*EntityServiceSqlTest" }) public class SqlDaoServiceTestSuite { diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index 220aaf32e7..7df5c6f868 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -47,6 +48,11 @@ import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.NumericFilterPredicate; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.EntityTypeFilter; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.attributes.AttributesService; import java.util.ArrayList; @@ -119,6 +125,61 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { Assert.assertEquals(0, count); } + @Test + public void testCountHierarchicalEntitiesByQuery() { + List assets = new ArrayList<>(); + List devices = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("Asset" + i); + asset.setType("type" + i); + asset.setLabel("AssetLabel" + i); + asset = assetService.saveAsset(asset); + assets.add(asset); + EntityRelation er = new EntityRelation(); + er.setFrom(tenantId); + er.setTo(asset.getId()); + er.setType("Manages"); + er.setTypeGroup(RelationTypeGroup.COMMON); + relationService.saveRelation(tenantId, er); + for (int j = 0; j < 5; j++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("A" + i + "Device" + j); + device.setType("default"); + device.setLabel("testLabel" + (int) (Math.random() * 1000)); + device = deviceService.saveDevice(device); + devices.add(device); + er = new EntityRelation(); + er.setFrom(asset.getId()); + er.setTo(device.getId()); + er.setType("Contains"); + er.setTypeGroup(RelationTypeGroup.COMMON); + relationService.saveRelation(tenantId, er); + } + } + + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setRootEntity(tenantId); + filter.setDirection(EntitySearchDirection.FROM); + + EntityCountQuery countQuery = new EntityCountQuery(filter); + + long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); + Assert.assertEquals(30, count); + + filter.setFilters(Collections.singletonList(new EntityTypeFilter("Contains", Collections.singletonList(EntityType.DEVICE)))); + count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); + Assert.assertEquals(25, count); + + filter.setRootEntity(devices.get(0).getId()); + filter.setDirection(EntitySearchDirection.TO); + filter.setFilters(Collections.singletonList(new EntityTypeFilter("Manages", Collections.singletonList(EntityType.TENANT)))); + count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); + Assert.assertEquals(1, count); + } + @Test public void testSimpleFindEntityDataByQuery() { List devices = new ArrayList<>(); @@ -187,14 +248,14 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { List devices = new ArrayList<>(); List temperatures = new ArrayList<>(); List highTemperatures = new ArrayList<>(); - for (int i=0;i<67;i++) { + for (int i = 0; i < 67; i++) { Device device = new Device(); device.setTenantId(tenantId); - device.setName("Device"+i); + device.setName("Device" + i); device.setType("default"); - device.setLabel("testLabel"+(int)(Math.random()*1000)); + device.setLabel("testLabel" + (int) (Math.random() * 1000)); devices.add(deviceService.saveDevice(device)); - long temperature = (long)(Math.random()*100); + long temperature = (long) (Math.random() * 100); temperatures.add(temperature); if (temperature > 45) { highTemperatures.add(temperature); @@ -202,7 +263,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } List>> attributeFutures = new ArrayList<>(); - for (int i=0;i