From ca435136899d715afee6ff549a588c269fe1bd9c Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 17 Jun 2020 17:20:36 +0300 Subject: [PATCH] Implementation of Hierarchical queries --- .../query/DefaultEntityQueryRepository.java | 262 ++++++++++----- .../dao/sql/query/EntityKeyMapping.java | 35 +- .../dao/service/BaseEntityServiceTest.java | 309 ++++++++++++++++-- 3 files changed, 487 insertions(+), 119 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index 8b9c6d3a97..8827dff8cb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,11 +20,14 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Repository; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.UUIDConverter; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; import org.thingsboard.server.common.data.query.AssetTypeFilter; +import org.thingsboard.server.common.data.query.DeviceSearchQueryFilter; import org.thingsboard.server.common.data.query.DeviceTypeFilter; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; @@ -35,6 +38,7 @@ import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.common.data.query.EntityFilterType; import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.query.EntityNameFilter; +import org.thingsboard.server.common.data.query.EntitySearchQueryFilter; import org.thingsboard.server.common.data.query.EntityViewTypeFilter; import org.thingsboard.server.common.data.query.RelationsQueryFilter; import org.thingsboard.server.common.data.query.SingleEntityFilter; @@ -57,7 +61,7 @@ import java.util.stream.Collectors; @Repository @Slf4j public class DefaultEntityQueryRepository implements EntityQueryRepository { - + //TODO: rafactoring to protect from SQL injections; private static final Map entityTableMap = new HashMap<>(); static { @@ -70,6 +74,22 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { entityTableMap.put(EntityType.TENANT, "tenant"); } + public static final String HIERARCHICAL_QUERY_TEMPLATE = " FROM (WITH RECURSIVE related_entities(from_id, from_type, to_id, to_type, relation_type, lvl) AS (" + + " SELECT from_id, from_type, to_id, to_type, relation_type, 1 as lvl" + + " FROM relation" + + " WHERE $in_id = '%s' and $in_type = '%s' and relation_type_group = 'COMMON'" + + " UNION ALL" + + " SELECT r.from_id, r.from_type, r.to_id, r.to_type, r.relation_type, lvl + 1" + + " FROM relation r" + + " INNER JOIN related_entities re ON" + + " r.$in_id = re.$out_id and r.$in_type = re.$out_type and" + + " relation_type_group = 'COMMON' %s)" + + " SELECT re.$out_id entity_id, re.$out_type entity_type, re.lvl lvl" + + " from related_entities re" + + " %s ) entity"; + public static final String HIERARCHICAL_TO_QUERY_TEMPLATE = HIERARCHICAL_QUERY_TEMPLATE.replace("$in", "to").replace("$out", "from"); + public static final String HIERARCHICAL_FROM_QUERY_TEMPLATE = HIERARCHICAL_QUERY_TEMPLATE.replace("$in", "from").replace("$out", "to"); + @PersistenceContext private EntityManager entityManager; @@ -109,13 +129,19 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { String entityWhereClause = this.buildEntityWhere(tenantId, customerId, query.getEntityFilter(), entityFieldsFiltersMapping, entityType); - String latestJoins = EntityKeyMapping.buildLatestJoins(entityType, allLatestMappings); + String latestJoins = EntityKeyMapping.buildLatestJoins(query.getEntityFilter(), entityType, allLatestMappings); String whereClause = this.buildWhere(selectionMapping, latestFiltersMapping, pageLink.getTextSearch()); String entityFieldsSelection = EntityKeyMapping.buildSelections(entityFieldsSelectionMapping); + String entityTypeStr; + if (query.getEntityFilter().getType().equals(EntityFilterType.RELATIONS_QUERY)) { + entityTypeStr = "e.entity_type"; + } else { + entityTypeStr = "'" + entityType.name() + "'"; + } if (!StringUtils.isEmpty(entityFieldsSelection)) { - entityFieldsSelection = String.format("e.id, '%s', %s", entityType.name(), entityFieldsSelection); + entityFieldsSelection = String.format("e.id, %s, %s", entityTypeStr, entityFieldsSelection); } else { - entityFieldsSelection = String.format("e.id, '%s'", entityType.name()); + entityFieldsSelection = String.format("e.id, %s", entityTypeStr); } String latestSelection = EntityKeyMapping.buildSelections(latestSelectionMapping); String topSelection = "entities.*"; @@ -175,16 +201,19 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } private String buildPermissionQuery(EntityFilter entityFilter, TenantId tenantId, CustomerId customerId, EntityType entityType) { - if (entityFilter.getType().equals(EntityFilterType.RELATIONS_QUERY)) { - return String.format("e.tenant_id='%s' and e.customer_id='%s'", UUIDConverter.fromTimeUUID(tenantId.getId()), UUIDConverter.fromTimeUUID(customerId.getId())); - } else { - if (entityType == EntityType.TENANT) { - return String.format("e.id='%s'", UUIDConverter.fromTimeUUID(tenantId.getId())); - } else if (entityType == EntityType.CUSTOMER) { - return String.format("e.tenant_id='%s' and e.id='%s'", UUIDConverter.fromTimeUUID(tenantId.getId()), UUIDConverter.fromTimeUUID(customerId.getId())); - } else { + switch (entityFilter.getType()) { + case RELATIONS_QUERY: + case DEVICE_SEARCH_QUERY: + case ASSET_SEARCH_QUERY: return String.format("e.tenant_id='%s' and e.customer_id='%s'", UUIDConverter.fromTimeUUID(tenantId.getId()), UUIDConverter.fromTimeUUID(customerId.getId())); - } + default: + if (entityType == EntityType.TENANT) { + return String.format("e.id='%s'", UUIDConverter.fromTimeUUID(tenantId.getId())); + } else if (entityType == EntityType.CUSTOMER) { + return String.format("e.tenant_id='%s' and e.id='%s'", UUIDConverter.fromTimeUUID(tenantId.getId()), UUIDConverter.fromTimeUUID(customerId.getId())); + } else { + return String.format("e.tenant_id='%s' and e.customer_id='%s'", UUIDConverter.fromTimeUUID(tenantId.getId()), UUIDConverter.fromTimeUUID(customerId.getId())); + } } } @@ -201,6 +230,8 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { case ENTITY_VIEW_TYPE: return this.typeQuery(entityFilter); case RELATIONS_QUERY: + case DEVICE_SEARCH_QUERY: + case ASSET_SEARCH_QUERY: return ""; default: throw new RuntimeException("Not implemented!"); @@ -211,84 +242,42 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { switch (entityFilter.getType()) { case RELATIONS_QUERY: return relationQuery((RelationsQueryFilter) entityFilter); + case DEVICE_SEARCH_QUERY: + DeviceSearchQueryFilter deviceQuery = (DeviceSearchQueryFilter) entityFilter; + return entitySearchQuery(deviceQuery, EntityType.DEVICE, deviceQuery.getDeviceTypes()); + case ASSET_SEARCH_QUERY: + AssetSearchQueryFilter assetQuery = (AssetSearchQueryFilter) entityFilter; + return entitySearchQuery(assetQuery, EntityType.ASSET, assetQuery.getAssetTypes()); default: return entityTableMap.get(entityType); } } - private String relationQuery(RelationsQueryFilter entityFilter) { + private String entitySearchQuery(EntitySearchQueryFilter entityFilter, EntityType entityType, List types) { EntityId rootId = entityFilter.getRootEntity(); - String lvlFilter = entityFilter.getMaxLevel() > 0 ? ("and lvl <= " + (entityFilter.getMaxLevel() - 1)) : ""; - //TODO: refactoring - String selectFields = " select CASE\n" + - " WHEN entity.entity_type = 'DEVICE'\n" + - " THEN (select tenant_id from device where id = entity_id)\n" + - " WHEN entity.entity_type = 'ASSET'\n" + - " THEN (select tenant_id from asset where id = entity_id)\n" + - " WHEN entity.entity_type = 'CUSTOMER'\n" + - " THEN (select tenant_id from customer where id = entity_id)\n" + - " WHEN entity.entity_type = 'TENANT'\n" + - " THEN entity_id\n" + - " END as tenant_id,\n" + - " CASE\n" + - " WHEN entity.entity_type = 'DEVICE'\n" + - " THEN (select customer_id from device where id = entity_id)\n" + - " WHEN entity.entity_type = 'ASSET'\n" + - " THEN (select customer_id from asset where id = entity_id)\n" + - " WHEN entity.entity_type = 'CUSTOMER'\n" + - " THEN entity_id\n" + - " WHEN entity.entity_type = 'TENANT'\n" + - " THEN '1b21dd2138140008080808080808080'\n" + - " END as customer_id,\n" + - " entity.entity_id as id,\n" + - " CASE\n" + - " WHEN entity.entity_type = 'DEVICE'\n" + - " THEN (select type from device where id = entity_id)\n" + - " WHEN entity.entity_type = 'ASSET' THEN (select type from asset where id = entity_id)\n" + - " ELSE entity.entity_type\n" + - " END as type,\n" + - " CASE\n" + - " WHEN entity.entity_type = 'DEVICE'\n" + - " THEN (select name from device where id = entity_id)\n" + - " WHEN entity.entity_type = 'ASSET' THEN (select name from asset where id = entity_id)\n" + - " WHEN entity.entity_type = 'CUSTOMER'\n" + - " THEN (select title from customer where id = entity_id)\n" + - " WHEN entity.entity_type = 'TENANT'\n" + - " THEN (select title from tenant where id = entity_id)\n" + - " ELSE entity.entity_type\n" + - " END as name,\n" + - " CASE\n" + - " WHEN entity.entity_type = 'DEVICE'\n" + - " THEN (select label from device where id = entity_id)\n" + - " WHEN entity.entity_type = 'ASSET' THEN (select label from asset where id = entity_id)\n" + - " WHEN entity.entity_type = 'CUSTOMER'\n" + - " THEN (select title from customer where id = entity_id)\n" + - " WHEN entity.entity_type = 'TENANT'\n" + - " THEN (select title from tenant where id = entity_id)\n" + - " ELSE entity.entity_type\n" + - " END as label,\n" + - " entity.entity_type as entity_type"; - - String fromTemplate = " FROM (WITH RECURSIVE related_entities(from_id, from_type, to_id, to_type, relation_type, lvl) AS (" + - " SELECT from_id, from_type, to_id, to_type, relation_type, 1 as lvl" + - " FROM relation" + - " WHERE $in_id = '%s' and $in_type = '%s' and relation_type_group = 'COMMON'" + - " UNION ALL" + - " SELECT r.from_id, r.from_type, r.to_id, r.to_type, r.relation_type, lvl + 1" + - " FROM relation r" + - " INNER JOIN related_entities re ON" + - " r.$in_id = re.$out_id and r.$in_type = re.$out_type and" + - " relation_type_group = 'COMMON' %s)" + - " SELECT re.$out_id entity_id, re.$out_type entity_type, re.lvl lvl" + - " from related_entities re" + - " %s ) entity"; + //TODO: fetch last level only. + String lvlFilter = getLvlFilter(entityFilter.getMaxLevel()); + String selectFields = "SELECT tenant_id, customer_id, id, type, name, label FROM " + entityType.name() + " WHERE id in ( SELECT entity_id"; + String from = getQueryTemplate(entityFilter.getDirection()); - String from; - if (entityFilter.getDirection().equals(EntitySearchDirection.FROM)) { - from = fromTemplate.replace("$in", "from").replace("$out", "to"); - } else { - from = fromTemplate.replace("$in", "to").replace("$out", "from"); + String whereFilter = " WHERE " + " re.relation_type = '" + entityFilter.getRelationType() + "'" + + " AND re.to_type = '" + entityType.name() + "'"; + from = String.format(from, UUIDConverter.fromTimeUUID(rootId.getId()), rootId.getEntityType().name(), lvlFilter, whereFilter); + String query = "( " + selectFields + from + ")"; + if (types != null && !types.isEmpty()) { + query += " and type in (" + types.stream().map(type -> "'" + type + "'").collect(Collectors.joining(", ")) + ")"; } + query += " )"; + return query; + } + + private String relationQuery(RelationsQueryFilter entityFilter) { + EntityId rootId = entityFilter.getRootEntity(); + String lvlFilter = getLvlFilter(entityFilter.getMaxLevel()); + String selectFields = getSelectTenantId() + ", " + getSelectCustomerId() + ", " + + " entity.entity_id as id," + getSelectType() + ", " + getSelectName() + ", " + + getSelectLabel() + ", entity.entity_type as entity_type"; + String from = getQueryTemplate(entityFilter.getDirection()); StringBuilder whereFilter; if (entityFilter.getFilters() != null && !entityFilter.getFilters().isEmpty()) { @@ -302,13 +291,13 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { whereFilter.append(" AND "); } String relationType = etf.getRelationType(); - String types = etf.getEntityTypes().stream().map(type -> "'" + type + "'").collect(Collectors.joining(", ")); + String entityTypes = etf.getEntityTypes().stream().map(type -> "'" + type + "'").collect(Collectors.joining(", ")); if (!single) { whereFilter.append(" ("); } whereFilter.append(" re.relation_type = '").append(relationType).append("' and re.") .append(entityFilter.getDirection().equals(EntitySearchDirection.FROM) ? "to" : "from") - .append("_type in (").append(types).append(")"); + .append("_type in (").append(entityTypes).append(")"); if (!single) { whereFilter.append(" )"); } @@ -320,6 +309,107 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return "( " + selectFields + from + ")"; } + private String getLvlFilter(int maxLevel) { + return maxLevel > 0 ? ("and lvl <= " + (maxLevel - 1)) : ""; + } + + private String getQueryTemplate(EntitySearchDirection direction) { + String from; + if (direction.equals(EntitySearchDirection.FROM)) { + from = HIERARCHICAL_FROM_QUERY_TEMPLATE; + } else { + from = HIERARCHICAL_TO_QUERY_TEMPLATE; + } + return from; + } + + private String getSelectTenantId() { + return "SELECT CASE" + + " WHEN entity.entity_type = 'TENANT' THEN entity_id" + + " WHEN entity.entity_type = 'CUSTOMER'" + + " THEN (select tenant_id from customer where id = entity_id)" + + " WHEN entity.entity_type = 'USER'" + + " THEN (select tenant_id from tb_user where id = entity_id)" + + " WHEN entity.entity_type = 'DASHBOARD'" + + " THEN (select tenant_id from dashboard where id = entity_id)" + + " WHEN entity.entity_type = 'ASSET'" + + " THEN (select tenant_id from asset where id = entity_id)" + + " WHEN entity.entity_type = 'DEVICE'" + + " THEN (select tenant_id from device where id = entity_id)" + + " WHEN entity.entity_type = 'ENTITY_VIEW'" + + " THEN (select tenant_id from entity_view where id = entity_id)" + + " END as tenant_id"; + } + + private String getSelectCustomerId() { + return "CASE" + + " WHEN entity.entity_type = 'TENANT'" + + " THEN '" + UUIDConverter.fromTimeUUID(TenantId.NULL_UUID) + "'" + + " WHEN entity.entity_type = 'CUSTOMER' THEN entity_id" + + " WHEN entity.entity_type = 'USER'" + + " THEN (select customer_id from tb_user where id = entity_id)" + + " WHEN entity.entity_type = 'DASHBOARD'" + + //TODO: parse assigned customers or use contains? + " THEN NULL" + + " WHEN entity.entity_type = 'ASSET'" + + " THEN (select customer_id from asset where id = entity_id)" + + " WHEN entity.entity_type = 'DEVICE'" + + " THEN (select customer_id from device where id = entity_id)" + + " WHEN entity.entity_type = 'ENTITY_VIEW'" + + " THEN (select customer_id from entity_view where id = entity_id)" + + " END as customer_id"; + } + + private String getSelectName() { + return " CASE" + + " WHEN entity.entity_type = 'TENANT'" + + " THEN (select title from tenant where id = entity_id)" + + " WHEN entity.entity_type = 'CUSTOMER' " + + " THEN (select title from customer where id = entity_id)" + + " WHEN entity.entity_type = 'USER'" + + " THEN (select CONCAT (first_name, ' ', last_name) from tb_user where id = entity_id)" + + " WHEN entity.entity_type = 'DASHBOARD'" + + " THEN (select title from dashboard where id = entity_id)" + + " WHEN entity.entity_type = 'ASSET'" + + " THEN (select name from asset where id = entity_id)" + + " WHEN entity.entity_type = 'DEVICE'" + + " THEN (select name from device where id = entity_id)" + + " WHEN entity.entity_type = 'ENTITY_VIEW'" + + " THEN (select name from entity_view where id = entity_id)" + + " END as name"; + } + + private String getSelectType() { + return " CASE" + + " WHEN entity.entity_type = 'USER'" + + " THEN (select authority from tb_user where id = entity_id)" + + " WHEN entity.entity_type = 'ASSET'" + + " THEN (select type from asset where id = entity_id)" + + " WHEN entity.entity_type = 'DEVICE'" + + " THEN (select type from device where id = entity_id)" + + " WHEN entity.entity_type = 'ENTITY_VIEW'" + + " THEN (select type from entity_view where id = entity_id)" + + " ELSE entity.entity_type END as type"; + } + + private String getSelectLabel() { + return " CASE" + + " WHEN entity.entity_type = 'TENANT'" + + " THEN (select title from tenant where id = entity_id)" + + " WHEN entity.entity_type = 'CUSTOMER' " + + " THEN (select title from customer where id = entity_id)" + + " WHEN entity.entity_type = 'USER'" + + " THEN (select CONCAT (first_name, ' ', last_name) from tb_user where id = entity_id)" + + " WHEN entity.entity_type = 'DASHBOARD'" + + " THEN (select title from dashboard where id = entity_id)" + + " WHEN entity.entity_type = 'ASSET'" + + " THEN (select label from asset where id = entity_id)" + + " WHEN entity.entity_type = 'DEVICE'" + + " THEN (select label from device where id = entity_id)" + + " WHEN entity.entity_type = 'ENTITY_VIEW'" + + " THEN (select name from entity_view where id = entity_id)" + + " END as label"; + } private String buildWhere (List selectionMapping, List latestFiltersMapping, String searchText) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java index f764c28f91..a71b6e8d85 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java @@ -22,6 +22,8 @@ import org.thingsboard.server.common.data.query.BooleanFilterPredicate; import org.thingsboard.server.common.data.query.ComplexFilterPredicate; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.query.EntityFilterType; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.FilterPredicateType; @@ -43,6 +45,7 @@ import java.util.stream.Stream; public class EntityKeyMapping { public static final Map entityFieldColumnMap = new HashMap<>(); + static { entityFieldColumnMap.put("createdTime", "id"); entityFieldColumnMap.put("name", "name"); @@ -107,14 +110,20 @@ public class EntityKeyMapping { } } - public String toLatestJoin(EntityType entityType) { + public String toLatestJoin(EntityFilter entityFilter, EntityType entityType) { + String entityTypeStr; + if (entityFilter.getType().equals(EntityFilterType.RELATIONS_QUERY)) { + entityTypeStr = "entities.entity_type"; + } else { + entityTypeStr = "'" + entityType.name() + "'"; + } String join = hasFilter() ? "left join" : "left outer join"; if (entityKey.getType().equals(EntityKeyType.TIME_SERIES)) { // TODO: throw new RuntimeException("Not implemented!"); } else { - String query = String.format("%s attribute_kv %s ON %s.entity_id=entities.id AND %s.entity_type='%s' AND %s.attribute_key='%s'", - join, alias, alias, alias, entityType.name(), alias, entityKey.getKey()); + String query = String.format("%s attribute_kv %s ON %s.entity_id=entities.id AND %s.entity_type=%s AND %s.attribute_key='%s'", + join, alias, alias, alias, entityTypeStr, alias, entityKey.getKey()); if (!entityKey.getType().equals(EntityKeyType.ATTRIBUTE)) { String scope; if (entityKey.getType().equals(EntityKeyType.CLIENT_ATTRIBUTE)) { @@ -135,8 +144,8 @@ public class EntityKeyMapping { Collectors.joining(", ")); } - public static String buildLatestJoins(EntityType entityType, List latestMappings) { - return latestMappings.stream().map(mapping -> mapping.toLatestJoin(entityType)).collect( + public static String buildLatestJoins(EntityFilter entityFilter, EntityType entityType, List latestMappings) { + return latestMappings.stream().map(mapping -> mapping.toLatestJoin(entityFilter, entityType)).collect( Collectors.joining(" ")); } @@ -207,7 +216,7 @@ public class EntityKeyMapping { if (mapping.getEntityKey().getType().equals(EntityKeyType.ENTITY_FIELD)) { index++; } else { - index +=2; + index += 2; } } if (!filters.isEmpty()) { @@ -220,7 +229,7 @@ public class EntityKeyMapping { mapping.setSelection(false); mapping.setEntityKey(filterField); mappings.add(mapping); - index +=1; + index += 1; } } @@ -253,7 +262,7 @@ public class EntityKeyMapping { private String buildPredicateQuery(String alias, EntityKey key, KeyFilterPredicate predicate) { if (predicate.getType().equals(FilterPredicateType.COMPLEX)) { - return this.buildComplexPredicateQuery(alias, key, (ComplexFilterPredicate)predicate); + return this.buildComplexPredicateQuery(alias, key, (ComplexFilterPredicate) predicate); } else { return this.buildSimplePredicateQuery(alias, key, predicate); } @@ -270,10 +279,10 @@ public class EntityKeyMapping { if (predicate.getType().equals(FilterPredicateType.NUMERIC)) { if (key.getType().equals(EntityKeyType.ENTITY_FIELD)) { String column = entityFieldColumnMap.get(key.getKey()); - return this.buildNumericPredicateQuery(alias + "." + column, (NumericFilterPredicate)predicate); + return this.buildNumericPredicateQuery(alias + "." + column, (NumericFilterPredicate) predicate); } else { - String longQuery = this.buildNumericPredicateQuery(alias + ".long_v", (NumericFilterPredicate)predicate); - String doubleQuery = this.buildNumericPredicateQuery(alias + ".dbl_v", (NumericFilterPredicate)predicate); + String longQuery = this.buildNumericPredicateQuery(alias + ".long_v", (NumericFilterPredicate) predicate); + String doubleQuery = this.buildNumericPredicateQuery(alias + ".dbl_v", (NumericFilterPredicate) predicate); return String.format("(%s or %s)", longQuery, doubleQuery); } } else { @@ -285,9 +294,9 @@ public class EntityKeyMapping { } String field = alias + "." + column; if (predicate.getType().equals(FilterPredicateType.STRING)) { - return this.buildStringPredicateQuery(field, (StringFilterPredicate)predicate); + return this.buildStringPredicateQuery(field, (StringFilterPredicate) predicate); } else { - return this.buildBooleanPredicateQuery(field, (BooleanFilterPredicate)predicate); + return this.buildBooleanPredicateQuery(field, (BooleanFilterPredicate) predicate); } } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index 6d5bb92490..21158d1582 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetSearchQuery; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -36,6 +37,8 @@ import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; +import org.thingsboard.server.common.data.query.DeviceSearchQueryFilter; import org.thingsboard.server.common.data.query.DeviceTypeFilter; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; @@ -55,6 +58,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.attributes.AttributesService; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; @@ -128,6 +132,280 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { public void testCountHierarchicalEntitiesByQuery() { List assets = new ArrayList<>(); List devices = new ArrayList<>(); + createTestHierarchy(assets, devices, new ArrayList<>(), new ArrayList<>(), new ArrayList<>(), new ArrayList<>()); + + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setRootEntity(tenantId); + filter.setDirection(EntitySearchDirection.FROM); + + EntityCountQuery countQuery = new EntityCountQuery(filter); + + long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); + Assert.assertEquals(30, count); + + filter.setFilters(Collections.singletonList(new EntityTypeFilter("Contains", Collections.singletonList(EntityType.DEVICE)))); + count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); + Assert.assertEquals(25, count); + + filter.setRootEntity(devices.get(0).getId()); + filter.setDirection(EntitySearchDirection.TO); + filter.setFilters(Collections.singletonList(new EntityTypeFilter("Manages", Collections.singletonList(EntityType.TENANT)))); + count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); + Assert.assertEquals(1, count); + + DeviceSearchQueryFilter filter2 = new DeviceSearchQueryFilter(); + filter2.setRootEntity(tenantId); + filter2.setDirection(EntitySearchDirection.FROM); + filter2.setRelationType("Contains"); + + countQuery = new EntityCountQuery(filter2); + + count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); + Assert.assertEquals(25, count); + + filter2.setDeviceTypes(Arrays.asList("default0", "default1")); + count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); + Assert.assertEquals(10, count); + + filter2.setRootEntity(devices.get(0).getId()); + filter2.setDirection(EntitySearchDirection.TO); + count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); + Assert.assertEquals(0, count); + + AssetSearchQueryFilter filter3 = new AssetSearchQueryFilter(); + filter3.setRootEntity(tenantId); + filter3.setDirection(EntitySearchDirection.FROM); + filter3.setRelationType("Manages"); + + countQuery = new EntityCountQuery(filter3); + + count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); + Assert.assertEquals(5, count); + + filter3.setAssetTypes(Arrays.asList("type0", "type1")); + count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); + Assert.assertEquals(2, count); + + filter3.setRootEntity(devices.get(0).getId()); + filter3.setDirection(EntitySearchDirection.TO); + count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); + Assert.assertEquals(0, count); + } + + @Test + public void testHierarchicalFindEntityDataWithAttributesByQuery() throws ExecutionException, InterruptedException { + List assets = new ArrayList<>(); + List devices = new ArrayList<>(); + List temperatures = new ArrayList<>(); + List highTemperatures = new ArrayList<>(); + createTestHierarchy(assets, devices, new ArrayList<>(), new ArrayList<>(), temperatures, highTemperatures); + + List>> attributeFutures = new ArrayList<>(); + for (int i = 0; i < devices.size(); i++) { + Device device = devices.get(i); + attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), DataConstants.CLIENT_SCOPE)); + } + Futures.successfulAsList(attributeFutures).get(); + + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setRootEntity(tenantId); + filter.setDirection(EntitySearchDirection.FROM); + filter.setFilters(Collections.singletonList(new EntityTypeFilter("Contains", Collections.singletonList(EntityType.DEVICE)))); + + EntityDataSortOrder sortOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC + ); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder); + List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); + + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); + PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + List loadedEntities = new ArrayList<>(data.getData()); + while (data.hasNext()) { + query = query.next(); + data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + loadedEntities.addAll(data.getData()); + } + Assert.assertEquals(25, loadedEntities.size()); + List loadedTemperatures = loadedEntities.stream().map(entityData -> + entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); + List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); + Assert.assertEquals(deviceTemperatures, loadedTemperatures); + + pageLink = new EntityDataPageLink(10, 0, null, sortOrder); + KeyFilter highTemperatureFilter = new KeyFilter(); + highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); + NumericFilterPredicate predicate = new NumericFilterPredicate(); + predicate.setValue(45); + predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + highTemperatureFilter.setPredicate(predicate); + List keyFilters = Collections.singletonList(highTemperatureFilter); + + query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + + data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + + loadedEntities = new ArrayList<>(data.getData()); + while (data.hasNext()) { + query = query.next(); + data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + loadedEntities.addAll(data.getData()); + } + Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); + + List loadedHighTemperatures = loadedEntities.stream().map(entityData -> + entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); + List deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); + + Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); + + deviceService.deleteDevicesByTenantId(tenantId); + } + + @Test + public void testHierarchicalFindDevicesWithAttributesByQuery() throws ExecutionException, InterruptedException { + List assets = new ArrayList<>(); + List devices = new ArrayList<>(); + List temperatures = new ArrayList<>(); + List highTemperatures = new ArrayList<>(); + createTestHierarchy(assets, devices, new ArrayList<>(), new ArrayList<>(), temperatures, highTemperatures); + + List>> attributeFutures = new ArrayList<>(); + for (int i = 0; i < devices.size(); i++) { + Device device = devices.get(i); + attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), DataConstants.CLIENT_SCOPE)); + } + Futures.successfulAsList(attributeFutures).get(); + + DeviceSearchQueryFilter filter = new DeviceSearchQueryFilter(); + filter.setRootEntity(tenantId); + filter.setDirection(EntitySearchDirection.FROM); + filter.setRelationType("Contains"); + + EntityDataSortOrder sortOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC + ); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder); + List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); + + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); + PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + List loadedEntities = new ArrayList<>(data.getData()); + while (data.hasNext()) { + query = query.next(); + data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + loadedEntities.addAll(data.getData()); + } + Assert.assertEquals(25, loadedEntities.size()); + List loadedTemperatures = loadedEntities.stream().map(entityData -> + entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); + List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); + Assert.assertEquals(deviceTemperatures, loadedTemperatures); + + pageLink = new EntityDataPageLink(10, 0, null, sortOrder); + KeyFilter highTemperatureFilter = new KeyFilter(); + highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); + NumericFilterPredicate predicate = new NumericFilterPredicate(); + predicate.setValue(45); + predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + highTemperatureFilter.setPredicate(predicate); + List keyFilters = Collections.singletonList(highTemperatureFilter); + + query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + + data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + + loadedEntities = new ArrayList<>(data.getData()); + while (data.hasNext()) { + query = query.next(); + data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + loadedEntities.addAll(data.getData()); + } + Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); + + List loadedHighTemperatures = loadedEntities.stream().map(entityData -> + entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); + List deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); + + Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); + + deviceService.deleteDevicesByTenantId(tenantId); + } + + @Test + public void testHierarchicalFindAssetsWithAttributesByQuery() throws ExecutionException, InterruptedException { + List assets = new ArrayList<>(); + List devices = new ArrayList<>(); + List consumptions = new ArrayList<>(); + List highConsumptions = new ArrayList<>(); + createTestHierarchy(assets, devices, consumptions, highConsumptions, new ArrayList<>(), new ArrayList<>()); + + List>> attributeFutures = new ArrayList<>(); + for (int i = 0; i < assets.size(); i++) { + Asset asset = assets.get(i); + attributeFutures.add(saveLongAttribute(asset.getId(), "consumption", consumptions.get(i), DataConstants.SERVER_SCOPE)); + } + Futures.successfulAsList(attributeFutures).get(); + + AssetSearchQueryFilter filter = new AssetSearchQueryFilter(); + filter.setRootEntity(tenantId); + filter.setDirection(EntitySearchDirection.FROM); + filter.setRelationType("Manages"); + + EntityDataSortOrder sortOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC + ); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder); + List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "consumption")); + + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); + PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + List loadedEntities = new ArrayList<>(data.getData()); + while (data.hasNext()) { + query = query.next(); + data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + loadedEntities.addAll(data.getData()); + } + Assert.assertEquals(5, loadedEntities.size()); + List loadedTemperatures = loadedEntities.stream().map(entityData -> + entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("consumption").getValue()).collect(Collectors.toList()); + List deviceTemperatures = consumptions.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); + Assert.assertEquals(deviceTemperatures, loadedTemperatures); + + pageLink = new EntityDataPageLink(10, 0, null, sortOrder); + KeyFilter highTemperatureFilter = new KeyFilter(); + highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "consumption")); + NumericFilterPredicate predicate = new NumericFilterPredicate(); + predicate.setValue(50); + predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + highTemperatureFilter.setPredicate(predicate); + List keyFilters = Collections.singletonList(highTemperatureFilter); + + query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + + data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + + loadedEntities = new ArrayList<>(data.getData()); + while (data.hasNext()) { + query = query.next(); + data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + loadedEntities.addAll(data.getData()); + } + Assert.assertEquals(highConsumptions.size(), loadedEntities.size()); + + List loadedHighTemperatures = loadedEntities.stream().map(entityData -> + entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("consumption").getValue()).collect(Collectors.toList()); + List deviceHighTemperatures = highConsumptions.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); + + Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); + + deviceService.deleteDevicesByTenantId(tenantId); + } + + private void createTestHierarchy(List assets, List devices, List consumptions, List highConsumptions, List temperatures, List highTemperatures) { for (int i = 0; i < 5; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); @@ -142,11 +420,16 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { er.setType("Manages"); er.setTypeGroup(RelationTypeGroup.COMMON); relationService.saveRelation(tenantId, er); + long consumption = (long) (Math.random() * 100); + consumptions.add(consumption); + if (consumption > 50) { + highConsumptions.add(consumption); + } for (int j = 0; j < 5; j++) { Device device = new Device(); device.setTenantId(tenantId); device.setName("A" + i + "Device" + j); - device.setType("default"); + device.setType("default" + j); device.setLabel("testLabel" + (int) (Math.random() * 1000)); device = deviceService.saveDevice(device); devices.add(device); @@ -156,27 +439,13 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { er.setType("Contains"); er.setTypeGroup(RelationTypeGroup.COMMON); relationService.saveRelation(tenantId, er); + long temperature = (long) (Math.random() * 100); + temperatures.add(temperature); + if (temperature > 45) { + highTemperatures.add(temperature); + } } } - - RelationsQueryFilter filter = new RelationsQueryFilter(); - filter.setRootEntity(tenantId); - filter.setDirection(EntitySearchDirection.FROM); - - EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(30, count); - - filter.setFilters(Collections.singletonList(new EntityTypeFilter("Contains", Collections.singletonList(EntityType.DEVICE)))); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(25, count); - - filter.setRootEntity(devices.get(0).getId()); - filter.setDirection(EntitySearchDirection.TO); - filter.setFilters(Collections.singletonList(new EntityTypeFilter("Manages", Collections.singletonList(EntityType.TENANT)))); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(1, count); } @Test