From 85728c0e248eb8669720316fc8a3e1287077cdc7 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 15 Mar 2024 17:26:31 +0200 Subject: [PATCH] added ownerName ENTITY_FIELD to EntityDataQuery --- .../controller/EntityQueryControllerTest.java | 104 ++++++++++++++++++ .../query/DefaultEntityQueryRepository.java | 30 +++-- .../dao/sql/query/EntityKeyMapping.java | 82 +++++++++----- 3 files changed, 182 insertions(+), 34 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index 67ad2cbe55..edd2a372c1 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -50,6 +50,7 @@ import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.NumericFilterPredicate; +import org.thingsboard.server.common.data.query.StringFilterPredicate; import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.service.DaoSqlTest; @@ -593,4 +594,107 @@ public class EntityQueryControllerTest extends AbstractControllerTest { assertThat(getErrorMessage(result)).contains("Invalid").contains("sort property"); } + @Test + public void testFindDevicesByOwnerNameAndOwnerType() throws Exception { + loginTenantAdmin(); + int numOfDevices = 3; + + for (int i = 0; i < numOfDevices; i++) { + Device device = new Device(); + String name = "Device" + i; + device.setName(name); + device.setType("default"); + + Device savedDevice = doPost("/api/device?accessToken=" + name, device, Device.class); + JsonNode content = JacksonUtil.toJsonNode("{\"alarmActiveTime\": 1" + i + "}"); + doPost("/api/plugins/telemetry/" + EntityType.DEVICE.name() + "/" + savedDevice.getUuidId() + "/SERVER_SCOPE", content) + .andExpect(status().isOk()); + } + + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceTypes(List.of("default")); + filter.setDeviceNameFilter(""); + + KeyFilter activeAlarmTimeFilter = getServerAttributeNumericGreaterThanKeyFilter("alarmActiveTime", 5); + KeyFilter tenantOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", TEST_TENANT_NAME); + KeyFilter wrongOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", "wrongName"); + KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); + KeyFilter customerOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "CUSTOMER"); + + EntityDataSortOrder sortOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC + ); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder); + List entityFields = List.of(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "ownerName"), + new EntityKey(EntityKeyType.ENTITY_FIELD, "ownerType")); + List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "alarmActiveTime")); + + // all devices with ownerName = TEST TENANT + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, List.of(activeAlarmTimeFilter, tenantOwnerNameFilter)); + checkEntitiesByQuery(query, numOfDevices, TEST_TENANT_NAME, "TENANT"); + + // all devices with wrong ownerName + EntityDataQuery wrongTenantNameQuery = new EntityDataQuery(filter, pageLink, entityFields, latestValues, List.of(activeAlarmTimeFilter, wrongOwnerNameFilter)); + checkEntitiesByQuery(wrongTenantNameQuery, 0, null, null); + + // all devices with owner type = TENANT + EntityDataQuery tenantEntitiesQuery = new EntityDataQuery(filter, pageLink, entityFields, latestValues, List.of(activeAlarmTimeFilter, tenantOwnerTypeFilter)); + checkEntitiesByQuery(tenantEntitiesQuery, numOfDevices, TEST_TENANT_NAME, "TENANT"); + + // all devices with owner type = CUSTOMER + EntityDataQuery customerEntitiesQuery = new EntityDataQuery(filter, pageLink, entityFields, latestValues, List.of(activeAlarmTimeFilter, customerOwnerTypeFilter)); + checkEntitiesByQuery(customerEntitiesQuery, 0, null, null); + } + + private void checkEntitiesByQuery(EntityDataQuery query, int expectedNumOfDevices, String expectedOwnerName, String expectedOwnerType) throws Exception { + Awaitility.await() + .alias("data by query") + .atMost(30, TimeUnit.SECONDS) + .until(() -> { + var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + var loadedEntities = new ArrayList<>(data.getData()); + return loadedEntities.size() == expectedNumOfDevices; + }); + if (expectedNumOfDevices == 0) { + return; + } + var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + var loadedEntities = new ArrayList<>(data.getData()); + + Assert.assertEquals(expectedNumOfDevices, loadedEntities.size()); + + for (int i = 0; i < expectedNumOfDevices; i++) { + var entity = loadedEntities.get(i); + String name = entity.getLatest().get(EntityKeyType.ENTITY_FIELD).getOrDefault("name", new TsValue(0, "Invalid")).getValue(); + String ownerName = entity.getLatest().get(EntityKeyType.ENTITY_FIELD).getOrDefault("ownerName", new TsValue(0, "Invalid")).getValue(); + String ownerType = entity.getLatest().get(EntityKeyType.ENTITY_FIELD).getOrDefault("ownerType", new TsValue(0, "Invalid")).getValue(); + String alarmActiveTime = entity.getLatest().get(EntityKeyType.ATTRIBUTE).getOrDefault("alarmActiveTime", new TsValue(0, "-1")).getValue(); + + Assert.assertEquals("Device" + i, name); + Assert.assertEquals( expectedOwnerName, ownerName); + Assert.assertEquals( expectedOwnerType, ownerType); + Assert.assertEquals("1" + i, alarmActiveTime); + } + } + + private KeyFilter getEntityFieldStringEqualToKeyFilter(String keyName, String value) { + KeyFilter tenantOwnerNameFilter = new KeyFilter(); + tenantOwnerNameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, keyName)); + StringFilterPredicate ownerNamePredicate = new StringFilterPredicate(); + ownerNamePredicate.setValue(FilterPredicateValue.fromString(value)); + ownerNamePredicate.setOperation(StringFilterPredicate.StringOperation.EQUAL); + tenantOwnerNameFilter.setPredicate(ownerNamePredicate); + return tenantOwnerNameFilter; + } + + private KeyFilter getServerAttributeNumericGreaterThanKeyFilter(String attribute, int value) { + KeyFilter numericFilter = new KeyFilter(); + numericFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, attribute)); + NumericFilterPredicate predicate = new NumericFilterPredicate(); + predicate.setValue(FilterPredicateValue.fromDouble(value)); + predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + numericFilter.setPredicate(predicate); + return numericFilter; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index d2b2567bd7..a7c11b70e7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -357,7 +357,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { List filterMapping = mappings.stream().filter(EntityKeyMapping::hasFilter) .collect(Collectors.toList()); - List entityFieldsFiltersMapping = filterMapping.stream().filter(mapping -> !mapping.isLatest()) + List entityFieldsFiltersMapping = filterMapping.stream().filter(mapping -> !mapping.isLatest() && mapping.getEntityKeyColumn() != null) .collect(Collectors.toList()); List allLatestMappings = mappings.stream().filter(EntityKeyMapping::isLatest) @@ -365,6 +365,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { String entityWhereClause = DefaultEntityQueryRepository.this.buildEntityWhere(ctx, query.getEntityFilter(), entityFieldsFiltersMapping); + String aliasWhereQuery = DefaultEntityQueryRepository.this.buildAliasWhereQuery(ctx, query.getEntityFilter(), selectionMapping, ""); String latestJoinsCnt = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings, true); String entityFieldsSelection = EntityKeyMapping.buildSelections(entityFieldsSelectionMapping, query.getEntityFilter().getType(), entityType); String entityTypeStr; @@ -385,7 +386,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { addEntityTableQuery(ctx, query.getEntityFilter()), entityWhereClause, latestJoinsCnt, - ""); + aliasWhereQuery); String countQuery = String.format("select count(id) %s", fromClauseCount); @@ -427,7 +428,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { List filterMapping = mappings.stream().filter(EntityKeyMapping::hasFilter) .collect(Collectors.toList()); - List entityFieldsFiltersMapping = filterMapping.stream().filter(mapping -> !mapping.isLatest()) + List entityFieldsFiltersMapping = filterMapping.stream().filter(mapping -> !mapping.isLatest() && mapping.getEntityKeyColumn() != null) .collect(Collectors.toList()); List allLatestMappings = mappings.stream().filter(EntityKeyMapping::isLatest) @@ -437,7 +438,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { String entityWhereClause = DefaultEntityQueryRepository.this.buildEntityWhere(ctx, query.getEntityFilter(), entityFieldsFiltersMapping); String latestJoinsCnt = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings, true); String latestJoinsData = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings, false); - String textSearchQuery = DefaultEntityQueryRepository.this.buildTextSearchQuery(ctx, selectionMapping, pageLink.getTextSearch()); + String aliasWhereQuery = DefaultEntityQueryRepository.this.buildAliasWhereQuery(ctx, query.getEntityFilter(), selectionMapping, pageLink.getTextSearch()); String entityFieldsSelection = EntityKeyMapping.buildSelections(entityFieldsSelectionMapping, query.getEntityFilter().getType(), entityType); String entityTypeStr; if (query.getEntityFilter().getType().equals(EntityFilterType.RELATIONS_QUERY)) { @@ -463,7 +464,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { addEntityTableQuery(ctx, query.getEntityFilter()), entityWhereClause, latestJoinsCnt, - textSearchQuery); + aliasWhereQuery); String fromClauseData = String.format("from (select %s from (select %s from %s e where %s) entities %s ) result %s", topSelection, @@ -471,7 +472,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { addEntityTableQuery(ctx, query.getEntityFilter()), entityWhereClause, latestJoinsData, - textSearchQuery); + aliasWhereQuery); if (!StringUtils.isEmpty(pageLink.getTextSearch())) { //Unfortunately, we need to sacrifice performance in case of full text search, because it is applied to all joined records. @@ -798,6 +799,21 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return from; } + private String buildAliasWhereQuery(QueryContext ctx, EntityFilter entityFilter, List selectionMapping, String searchText) { + List aliasFiltersMapping = selectionMapping.stream().filter(mapping -> !mapping.isLatest() && mapping.getEntityKeyColumn() == null) + .collect(Collectors.toList()); + String entityFieldsQuery = EntityKeyMapping.buildQuery(ctx, aliasFiltersMapping, entityFilter.getType()); + String searchTextQuery = buildTextSearchQuery(ctx, selectionMapping, searchText); + String result = ""; + if (!entityFieldsQuery.isEmpty()) { + result += " where (" + entityFieldsQuery + ")"; + } + if (!searchTextQuery.isEmpty()) { + result += (result.isEmpty() ? " where ": " and ") + "(" + searchTextQuery + ") "; + } + return result; + } + private String buildTextSearchQuery(QueryContext ctx, List selectionMapping, String searchText) { if (!StringUtils.isEmpty(searchText) && !selectionMapping.isEmpty()) { String sqlSearchText = "%" + searchText + "%"; @@ -809,7 +825,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } else { searchAliasesExpression = searchAliases.get(0); } - return String.format(" WHERE %s ILIKE :%s", searchAliasesExpression, "lowerSearchTextParam"); + return String.format(" %s ILIKE :%s", searchAliasesExpression, "lowerSearchTextParam"); } else { return ""; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java index b928e8790e..0f51ef72b2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java @@ -38,6 +38,7 @@ import org.thingsboard.server.dao.model.ModelConstants; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -48,6 +49,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static org.thingsboard.server.common.data.StringUtils.splitByCommaWithoutQuotes; +import static org.thingsboard.server.common.data.id.EntityId.NULL_UUID; +import static org.thingsboard.server.dao.sql.query.DefaultEntityQueryRepository.resolveEntityType; @Data public class EntityKeyMapping { @@ -55,6 +58,7 @@ public class EntityKeyMapping { private static final Map> allowedEntityFieldMap = new HashMap<>(); private static final Map entityFieldColumnMap = new HashMap<>(); private static final Map> aliases = new HashMap<>(); + private static final Map> propertiesFunctions = new EnumMap<>(EntityType.class); public static final String CREATED_TIME = "createdTime"; public static final String ENTITY_TYPE = "entityType"; @@ -75,6 +79,18 @@ public class EntityKeyMapping { public static final String PHONE = "phone"; public static final String ADDITIONAL_INFO = "additionalInfo"; public static final String RELATED_PARENT_ID = "parentId"; + public static final String OWNER_NAME = "ownerName"; + public static final String OWNER_TYPE = "ownerType"; + public static final String OWNER_NAME_SELECT_QUERY = "case when e.customer_id = '" + NULL_UUID + "' " + + "THEN (select title from tenant where id = e.tenant_id) " + + "ELSE (select title from customer where id = e.customer_id) END"; + public static final String OWNER_TYPE_SELECT_QUERY = "case when e.customer_id = '" + NULL_UUID + "' " + + "THEN 'TENANT' " + + "ELSE 'CUSTOMER' END"; + public static final Map ownerPropertiesFunctions = Map.of( + OWNER_NAME, OWNER_NAME_SELECT_QUERY, + OWNER_TYPE, OWNER_TYPE_SELECT_QUERY + ); public static final List typedEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, TYPE, ADDITIONAL_INFO); public static final List widgetEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME); @@ -140,6 +156,12 @@ public class EntityKeyMapping { aliases.put(EntityType.ENTITY_VIEW, commonEntityAliases); aliases.put(EntityType.WIDGETS_BUNDLE, commonEntityAliases); + propertiesFunctions.put(EntityType.DEVICE, ownerPropertiesFunctions); + propertiesFunctions.put(EntityType.ASSET, ownerPropertiesFunctions); + propertiesFunctions.put(EntityType.ENTITY_VIEW, ownerPropertiesFunctions); + propertiesFunctions.put(EntityType.USER, ownerPropertiesFunctions); + propertiesFunctions.put(EntityType.DASHBOARD, ownerPropertiesFunctions); + Map userEntityAliases = new HashMap<>(); userEntityAliases.put(TITLE, EMAIL); userEntityAliases.put(LABEL, EMAIL); @@ -150,6 +172,7 @@ public class EntityKeyMapping { private int index; private String alias; private boolean isLatest; + private String entityKeyColumn; private boolean isSelection; private boolean isSearchable; private boolean isSortOrder; @@ -179,12 +202,14 @@ public class EntityKeyMapping { if (entityKey.getKey().equals("entityType") && !filterType.equals(EntityFilterType.RELATIONS_QUERY)) { return String.format("'%s' as %s", entityType.name(), getValueAlias()); } else { - Set existingEntityFields = getExistingEntityFields(filterType, entityType); - String alias = getEntityFieldAlias(filterType, entityType); - if (existingEntityFields.contains(alias)) { - String column = entityFieldColumnMap.get(alias); - return String.format("cast(e.%s as varchar) as %s", column, getValueAlias()); + if (getEntityKeyColumn() != null) { + return String.format("cast(e.%s as varchar) as %s", getEntityKeyColumn(), getValueAlias()); } else { + Map entityPropertiesFunctions = propertiesFunctions.get(entityType); + String entityFieldAlias = getEntityFieldAlias(filterType, entityType); + if (entityPropertiesFunctions != null && entityPropertiesFunctions.containsKey(entityFieldAlias)) { + return String.format("%s as %s", entityPropertiesFunctions.get(entityFieldAlias), getValueAlias()); + } return String.format("'' as %s", getValueAlias()); } } @@ -234,7 +259,7 @@ public class EntityKeyMapping { public Stream toQueries(QueryContext ctx, EntityFilterType filterType) { if (hasFilter()) { - String keyAlias = entityKey.getType().equals(EntityKeyType.ENTITY_FIELD) ? "e" : alias; + String keyAlias = (entityKey.getType().equals(EntityKeyType.ENTITY_FIELD) && getEntityKeyColumn() != null) ? "e" : alias; return keyFilters.stream().map(keyFilter -> this.buildKeyQuery(ctx, keyAlias, keyFilter, filterType)); } else { @@ -315,6 +340,9 @@ public class EntityKeyMapping { } public static List prepareKeyMapping(EntityDataQuery query) { + EntityType entityType = resolveEntityType(query.getEntityFilter()); + EntityFilterType entityFilterType = query.getEntityFilter().getType(); + List entityFields = query.getEntityFields() != null ? query.getEntityFields() : Collections.emptyList(); List latestValues = query.getLatestValues() != null ? query.getLatestValues() : Collections.emptyList(); Map> filters = @@ -330,6 +358,7 @@ public class EntityKeyMapping { mapping.setSelection(true); mapping.setSearchable(!key.getKey().equals(ADDITIONAL_INFO)); mapping.setEntityKey(key); + mapping.setEntityKeyColumn(entityType, entityFilterType); return mapping; } ).collect(Collectors.toList()); @@ -361,6 +390,7 @@ public class EntityKeyMapping { sortOrderMapping.setEntityKey(sortOrderKey); sortOrderMapping.setSortOrder(true); sortOrderMapping.setIgnore(true); + sortOrderMapping.setEntityKeyColumn(entityType, entityFilterType); if (sortOrderKey.getType().equals(EntityKeyType.ENTITY_FIELD)) { entityFieldsMappings.add(sortOrderMapping); } else { @@ -398,6 +428,14 @@ public class EntityKeyMapping { return mappings; } + private void setEntityKeyColumn(EntityType entityType, EntityFilterType entityFilterType) { + Set existingEntityFields = getExistingEntityFields(entityFilterType, entityType); + String entityFieldAlias = getEntityFieldAlias(entityFilterType, entityType); + if (existingEntityFields.contains(entityFieldAlias)) { + entityKeyColumn = entityFieldColumnMap.get(entityFieldAlias); + } + } + public static List prepareEntityCountKeyMapping(EntityCountQuery query) { Map> filters = query.getKeyFilters() != null ? @@ -489,30 +527,20 @@ public class EntityKeyMapping { private String buildSimplePredicateQuery(QueryContext ctx, String alias, EntityKey key, KeyFilterPredicate predicate, EntityFilterType filterType) { if (key.getType().equals(EntityKeyType.ENTITY_FIELD)) { - Set existingEntityFields = getExistingEntityFields(filterType, ctx.getEntityType()); - String entityFieldAlias = getEntityFieldAlias(filterType, ctx.getEntityType()); - String column = null; - if (existingEntityFields.contains(entityFieldAlias)) { - column = entityFieldColumnMap.get(entityFieldAlias); - } - if (column != null) { - String field = alias + "." + column; - if (predicate.getType().equals(FilterPredicateType.NUMERIC)) { - return this.buildNumericPredicateQuery(ctx, field, (NumericFilterPredicate) predicate); - } else if (predicate.getType().equals(FilterPredicateType.STRING)) { - if (key.getKey().equals("entityType") && !filterType.equals(EntityFilterType.RELATIONS_QUERY)) { - field = ctx.getEntityType().toString(); - return this.buildStringPredicateQuery(ctx, field, (StringFilterPredicate) predicate) - .replace("lower(" + field, "lower('" + field + "'") - .replace(field + " ", "'" + field + "' "); - } else { - return this.buildStringPredicateQuery(ctx, field, (StringFilterPredicate) predicate); - } + String field = (getEntityKeyColumn() != null) ? alias + "." + getEntityKeyColumn() : alias; + if (predicate.getType().equals(FilterPredicateType.NUMERIC)) { + return this.buildNumericPredicateQuery(ctx, field, (NumericFilterPredicate) predicate); + } else if (predicate.getType().equals(FilterPredicateType.STRING)) { + if (key.getKey().equals("entityType") && !filterType.equals(EntityFilterType.RELATIONS_QUERY)) { + field = ctx.getEntityType().toString(); + return this.buildStringPredicateQuery(ctx, field, (StringFilterPredicate) predicate) + .replace("lower(" + field, "lower('" + field + "'") + .replace(field + " ", "'" + field + "' "); } else { - return this.buildBooleanPredicateQuery(ctx, field, (BooleanFilterPredicate) predicate); + return this.buildStringPredicateQuery(ctx, field, (StringFilterPredicate) predicate); } } else { - return null; + return this.buildBooleanPredicateQuery(ctx, field, (BooleanFilterPredicate) predicate); } } else { if (predicate.getType().equals(FilterPredicateType.NUMERIC)) {