Browse Source

add support for multi-root relation queries

pull/5255/head
vparomskiy 5 years ago
parent
commit
19d08428ff
  1. 3
      common/data/src/main/java/org/thingsboard/server/common/data/query/RelationsQueryFilter.java
  2. 12
      dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java
  3. 30
      dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java
  4. 4
      dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java
  5. 180
      dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java

3
common/data/src/main/java/org/thingsboard/server/common/data/query/RelationsQueryFilter.java

@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.relation.EntitySearchDirection;
import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter;
import java.util.List;
import java.util.Set;
@Data
public class RelationsQueryFilter implements EntityFilter {
@ -31,6 +32,8 @@ public class RelationsQueryFilter implements EntityFilter {
}
private EntityId rootEntity;
private boolean isMultiRoot;
private Set<EntityId> multiRootEntities;
private EntitySearchDirection direction;
private List<RelationEntityTypeFilter> filters;
private int maxLevel;

12
dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java

@ -42,6 +42,8 @@ import org.thingsboard.server.common.data.query.EntityCountQuery;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataPageLink;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityFilterType;
import org.thingsboard.server.common.data.query.RelationsQueryFilter;
import org.thingsboard.server.dao.alarm.AlarmService;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.dao.customer.CustomerService;
@ -228,6 +230,8 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe
throw new IncorrectParameterException("Query entity filter must be specified.");
} else if (query.getEntityFilter().getType() == null) {
throw new IncorrectParameterException("Query entity filter type must be specified.");
} else if (query.getEntityFilter().getType().equals(EntityFilterType.RELATIONS_QUERY)) {
validateRelationQuery((RelationsQueryFilter) query.getEntityFilter());
}
}
@ -246,4 +250,12 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe
}
}
private static void validateRelationQuery(RelationsQueryFilter queryFilter) {
if(queryFilter.isMultiRoot() && queryFilter.getMultiRootEntities().isEmpty()) {
throw new IncorrectParameterException("Multi-root relation query filter should contain root entities");
}
if(!queryFilter.isMultiRoot() && queryFilter.getRootEntity() == null) {
throw new IncorrectParameterException("Relation query filter root entity should not be blank");
}
}
}

30
dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java

@ -220,6 +220,8 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
" THEN (select additional_info from edge where id = entity_id)" +
" END as additional_info";
private static final String SELECT_RELATED_PARENT_ID = "entity.parentId AS parentId";
private static final String SELECT_API_USAGE_STATE = "(select aus.id, aus.created_time, aus.tenant_id, aus.entity_id, " +
"coalesce((select title from tenant where id = aus.entity_id), (select title from customer where id = aus.entity_id)) as name " +
"from api_usage_state as aus)";
@ -242,16 +244,16 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
private static final String HIERARCHICAL_QUERY_TEMPLATE = " FROM (WITH RECURSIVE related_entities(from_id, from_type, to_id, to_type, relation_type, lvl) AS (" +
" SELECT from_id, from_type, to_id, to_type, relation_type, 1 as lvl" +
" FROM relation" +
" WHERE $in_id = :relation_root_id and $in_type = :relation_root_type and relation_type_group = 'COMMON'" +
" WHERE $in_id $rootIdCondition and $in_type = :relation_root_type and relation_type_group = 'COMMON'" +
" UNION ALL" +
" SELECT r.from_id, r.from_type, r.to_id, r.to_type, r.relation_type, lvl + 1" +
" FROM relation r" +
" INNER JOIN related_entities re ON" +
" r.$in_id = re.$out_id and r.$in_type = re.$out_type and" +
" relation_type_group = 'COMMON' %s)" +
" SELECT re.$out_id entity_id, re.$out_type entity_type, max(re.lvl) lvl" +
" SELECT re.$out_id entity_id, re.$out_type entity_type, re.$in_id parentId, max(re.lvl) lvl" +
" from related_entities re" +
" %s GROUP BY entity_id, entity_type) entity";
" %s GROUP BY entity_id, entity_type, parentId) entity";
private static final String HIERARCHICAL_TO_QUERY_TEMPLATE = HIERARCHICAL_QUERY_TEMPLATE.replace("$in", "to").replace("$out", "from");
private static final String HIERARCHICAL_FROM_QUERY_TEMPLATE = HIERARCHICAL_QUERY_TEMPLATE.replace("$in", "from").replace("$out", "to");
@ -562,6 +564,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
+ (entityType.equals(EntityType.ENTITY_VIEW) ? "" : ", label ")
+ "FROM " + entityType.name() + " WHERE id in ( SELECT entity_id";
String from = getQueryTemplate(entityFilter.getDirection());
from = from.replace("$rootIdCondition", "= :relation_root_id");
String whereFilter = " WHERE";
if (!StringUtils.isEmpty(entityFilter.getRelationType())) {
ctx.addStringParameter("where_relation_type", entityFilter.getRelationType());
@ -604,11 +607,20 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
+ SELECT_TYPE + ", " + SELECT_NAME + ", " + SELECT_LABEL + ", " +
SELECT_FIRST_NAME + ", " + SELECT_LAST_NAME + ", " + SELECT_EMAIL + ", " + SELECT_REGION + ", " +
SELECT_TITLE + ", " + SELECT_COUNTRY + ", " + SELECT_STATE + ", " + SELECT_CITY + ", " +
SELECT_ADDRESS + ", " + SELECT_ADDRESS_2 + ", " + SELECT_ZIP + ", " + SELECT_PHONE + ", " + SELECT_ADDITIONAL_INFO +
SELECT_ADDRESS + ", " + SELECT_ADDRESS_2 + ", " + SELECT_ZIP + ", " + SELECT_PHONE + ", " +
SELECT_ADDITIONAL_INFO + ", " + SELECT_RELATED_PARENT_ID +
", entity.entity_type as entity_type";
String from = getQueryTemplate(entityFilter.getDirection());
ctx.addUuidParameter("relation_root_id", rootId.getId());
ctx.addStringParameter("relation_root_type", rootId.getEntityType().name());
if(entityFilter.isMultiRoot()) {
ctx.addUuidListParameter("relation_root_ids", entityFilter.getMultiRootEntities().stream().map(EntityId::getId).collect(Collectors.toList()));
ctx.addStringParameter("relation_root_type", entityFilter.getMultiRootEntities().stream().map(EntityId::getEntityType).findFirst().get().name());
from = from.replace("$rootIdCondition", "in (:relation_root_ids)");
} else {
ctx.addUuidParameter("relation_root_id", rootId.getId());
ctx.addStringParameter("relation_root_type", rootId.getEntityType().name());
from = from.replace("$rootIdCondition", "= :relation_root_id");
}
StringBuilder whereFilter = new StringBuilder();
@ -790,7 +802,11 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
case EDGE_SEARCH_QUERY:
return EntityType.EDGE;
case RELATIONS_QUERY:
return ((RelationsQueryFilter) entityFilter).getRootEntity().getEntityType();
RelationsQueryFilter rgf = (RelationsQueryFilter) entityFilter;
if(rgf.isMultiRoot()) {
return rgf.getMultiRootEntities().iterator().next().getEntityType();
}
return rgf.getRootEntity().getEntityType();
case API_USAGE_STATE:
return EntityType.API_USAGE_STATE;
default:

4
dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java

@ -72,6 +72,7 @@ public class EntityKeyMapping {
public static final String ZIP = "zip";
public static final String PHONE = "phone";
public static final String ADDITIONAL_INFO = "additionalInfo";
public static final String RELATED_PARENT_ID = "parentId";
public static final List<String> typedEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, TYPE, ADDITIONAL_INFO);
public static final List<String> widgetEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME);
@ -82,7 +83,7 @@ public class EntityKeyMapping {
public static final Set<String> apiUsageStateEntityFields = new HashSet<>(Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME));
public static final Set<String> commonEntityFieldsSet = new HashSet<>(commonEntityFields);
public static final Set<String> relationQueryEntityFieldsSet = new HashSet<>(Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, TYPE, LABEL, FIRST_NAME, LAST_NAME, EMAIL, REGION, TITLE, COUNTRY, STATE, CITY, ADDRESS, ADDRESS_2, ZIP, PHONE, ADDITIONAL_INFO));
public static final Set<String> relationQueryEntityFieldsSet = new HashSet<>(Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, TYPE, LABEL, FIRST_NAME, LAST_NAME, EMAIL, REGION, TITLE, COUNTRY, STATE, CITY, ADDRESS, ADDRESS_2, ZIP, PHONE, ADDITIONAL_INFO, RELATED_PARENT_ID));
static {
allowedEntityFieldMap.put(EntityType.DEVICE, new HashSet<>(labeledEntityFields));
@ -120,6 +121,7 @@ public class EntityKeyMapping {
entityFieldColumnMap.put(ZIP, ModelConstants.ZIP_PROPERTY);
entityFieldColumnMap.put(PHONE, ModelConstants.PHONE_PROPERTY);
entityFieldColumnMap.put(ADDITIONAL_INFO, ModelConstants.ADDITIONAL_INFO_PROPERTY);
entityFieldColumnMap.put(RELATED_PARENT_ID, RELATED_PARENT_ID);
Map<String, String> contactBasedAliases = new HashMap<>();
contactBasedAliases.put(NAME, TITLE);

180
dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java

@ -15,10 +15,12 @@
*/
package org.thingsboard.server.dao.service;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@ -35,6 +37,7 @@ import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.IdBased;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
@ -75,8 +78,11 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -373,6 +379,108 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
deviceService.deleteDevicesByTenantId(tenantId);
}
@Test
public void testCountHierarchicalEntitiesByMultiRootQuery() throws InterruptedException {
List<Asset> buildings = new ArrayList<>();
List<Asset> apartments = new ArrayList<>();
Map<String, Map<UUID, String>> entityNameByTypeMap = new HashMap<>();
Map<UUID, UUID> childParentRelationMap = new HashMap<>();
createMultiRootHierarchy(buildings, apartments, entityNameByTypeMap, childParentRelationMap);
RelationsQueryFilter filter = new RelationsQueryFilter();
filter.setMultiRoot(true);
filter.setMultiRootEntities(buildings.stream().map(IdBased::getId).collect(Collectors.toSet()));
filter.setDirection(EntitySearchDirection.FROM);
EntityCountQuery countQuery = new EntityCountQuery(filter);
long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery);
Assert.assertEquals(63, count);
filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("AptToHeat", Collections.singletonList(EntityType.DEVICE))));
count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery);
Assert.assertEquals(27, count);
filter.setMultiRootEntities(apartments.stream().map(IdBased::getId).collect(Collectors.toSet()));
filter.setDirection(EntitySearchDirection.TO);
filter.setFilters(Lists.newArrayList(
new RelationEntityTypeFilter("buildingToApt", Collections.singletonList(EntityType.ASSET)),
new RelationEntityTypeFilter("AptToEnergy", Collections.singletonList(EntityType.DEVICE))));
count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery);
Assert.assertEquals(9, count);
deviceService.deleteDevicesByTenantId(tenantId);
assetService.deleteAssetsByTenantId(tenantId);
}
@Test
public void testMultiRootHierarchicalFindEntityDataWithAttributesByQuery() throws ExecutionException, InterruptedException {
List<Asset> buildings = new ArrayList<>();
List<Asset> apartments = new ArrayList<>();
Map<String, Map<UUID, String>> entityNameByTypeMap = new HashMap<>();
Map<UUID, UUID> childParentRelationMap = new HashMap<>();
createMultiRootHierarchy(buildings, apartments, entityNameByTypeMap, childParentRelationMap);
RelationsQueryFilter filter = new RelationsQueryFilter();
filter.setMultiRoot(true);
filter.setMultiRootEntities(buildings.stream().map(IdBased::getId).collect(Collectors.toSet()));
filter.setDirection(EntitySearchDirection.FROM);
EntityDataSortOrder sortOrder = new EntityDataSortOrder(
new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC
);
EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
List<EntityKey> entityFields = Lists.newArrayList(
new EntityKey(EntityKeyType.ENTITY_FIELD, "name"),
new EntityKey(EntityKeyType.ENTITY_FIELD, "parentId"),
new EntityKey(EntityKeyType.ENTITY_FIELD, "type")
);
List<EntityKey> latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "status"));
KeyFilter onlineStatusFilter = new KeyFilter();
onlineStatusFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"));
StringFilterPredicate predicate = new StringFilterPredicate();
predicate.setOperation(StringOperation.ENDS_WITH);
predicate.setValue(FilterPredicateValue.fromString("_1"));
onlineStatusFilter.setPredicate(predicate);
List<KeyFilter> keyFilters = Collections.singletonList(onlineStatusFilter);
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters);
PageData<EntityData> data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
List<EntityData> loadedEntities = new ArrayList<>(data.getData());
while (data.hasNext()) {
query = query.next();
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities.addAll(data.getData());
}
long expectedEntitiesCnt = entityNameByTypeMap.entrySet()
.stream()
.filter(e -> !e.getKey().equals("building"))
.flatMap(e -> e.getValue().entrySet().stream())
.map(Map.Entry::getValue)
.filter(e -> StringUtils.endsWith(e, "_1"))
.count();
Assert.assertEquals(expectedEntitiesCnt, loadedEntities.size());
Map<UUID, UUID> actualRelations = new HashMap<>();
loadedEntities.forEach(ed -> {
UUID parentId = UUID.fromString(ed.getLatest().get(EntityKeyType.ENTITY_FIELD).get("parentId").getValue());
UUID entityId = ed.getEntityId().getId();
Assert.assertEquals(childParentRelationMap.get(entityId), parentId);
actualRelations.put(entityId, parentId);
String entityType = ed.getLatest().get(EntityKeyType.ENTITY_FIELD).get("type").getValue();
String actualEntityName = ed.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue();
String expectedEntityName = entityNameByTypeMap.get(entityType).get(entityId);
Assert.assertEquals(expectedEntityName, actualEntityName);
});
deviceService.deleteDevicesByTenantId(tenantId);
assetService.deleteAssetsByTenantId(tenantId);
}
@Test
public void testHierarchicalFindDevicesWithAttributesByQuery() throws ExecutionException, InterruptedException {
@ -1534,4 +1642,76 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
BasicTsKvEntry timeseries = new BasicTsKvEntry(42L, telemetryValue);
return timeseriesService.save(SYSTEM_TENANT_ID, entityId, timeseries);
}
private void createMultiRootHierarchy(List<Asset> buildings, List<Asset> apartments,
Map<String, Map<UUID, String>> entityNameByTypeMap,
Map<UUID, UUID> childParentRelationMap) throws InterruptedException {
for (int k = 0; k < 3; k++) {
Asset building = new Asset();
building.setTenantId(tenantId);
building.setName("Building _" + k);
building.setType("building");
building.setLabel("building label" + k);
building = assetService.saveAsset(building);
buildings.add(building);
entityNameByTypeMap.computeIfAbsent(building.getType(), n -> new HashMap<>()).put(building.getId().getId(), building.getName());
for (int i = 0; i < 3; i++) {
Asset asset = new Asset();
asset.setTenantId(tenantId);
asset.setName("Apt " + k + "_" + i);
asset.setType("apartment");
asset.setLabel("apartment " + i);
asset = assetService.saveAsset(asset);
//TO make sure devices have different created time
Thread.sleep(1);
entityNameByTypeMap.computeIfAbsent(asset.getType(), n -> new HashMap<>()).put(asset.getId().getId(), asset.getName());
apartments.add(asset);
EntityRelation er = new EntityRelation();
er.setFrom(building.getId());
er.setTo(asset.getId());
er.setType("buildingToApt");
er.setTypeGroup(RelationTypeGroup.COMMON);
relationService.saveRelation(tenantId, er);
childParentRelationMap.put(asset.getUuidId(), building.getUuidId());
for (int j = 0; j < 3; j++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("Heat" + k + "_" + i + "_" + j);
device.setType("heatmeter");
device.setLabel("heatmeter" + (int) (Math.random() * 1000));
device = deviceService.saveDevice(device);
//TO make sure devices have different created time
Thread.sleep(1);
entityNameByTypeMap.computeIfAbsent(device.getType(), n -> new HashMap<>()).put(device.getId().getId(), device.getName());
er = new EntityRelation();
er.setFrom(asset.getId());
er.setTo(device.getId());
er.setType("AptToHeat");
er.setTypeGroup(RelationTypeGroup.COMMON);
relationService.saveRelation(tenantId, er);
childParentRelationMap.put(device.getUuidId(), asset.getUuidId());
}
for (int j = 0; j < 3; j++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("Energy" + k + "_" + i + "_" + j);
device.setType("energymeter");
device.setLabel("energymeter" + (int) (Math.random() * 1000));
device = deviceService.saveDevice(device);
//TO make sure devices have different created time
Thread.sleep(1);
entityNameByTypeMap.computeIfAbsent(device.getType(), n -> new HashMap<>()).put(device.getId().getId(), device.getName());
er = new EntityRelation();
er.setFrom(asset.getId());
er.setTo(device.getId());
er.setType("AptToEnergy");
er.setTypeGroup(RelationTypeGroup.COMMON);
relationService.saveRelation(tenantId, er);
childParentRelationMap.put(device.getUuidId(), asset.getUuidId());
}
}
}
}
}

Loading…
Cancel
Save