Browse Source

Merge remote-tracking branch 'upstream/master' into bug/string-to-number-auto-convertion

pull/3169/head
vzikratyi 6 years ago
parent
commit
9c4d094075
  1. 12
      .travis.yml
  2. 66
      application/src/main/data/json/system/widget_bundles/maps.json
  3. 2
      application/src/main/data/upgrade/3.0.1/schema_update_to_uuid.sql
  4. 131
      dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java
  5. 74
      dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java
  6. 10
      dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java
  7. 4
      dao/src/main/resources/sql/schema-entities-idx.sql
  8. 722
      dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java
  9. 2
      ui-ngx/src/app/modules/home/components/attribute/add-attribute-dialog.component.ts
  10. 2
      ui-ngx/src/app/modules/home/components/entity/entity.component.ts
  11. 23
      ui-ngx/src/app/modules/home/components/filter/key-filter-dialog.component.html
  12. 33
      ui-ngx/src/app/modules/home/components/filter/key-filter-dialog.component.ts
  13. 8
      ui-ngx/src/app/modules/home/pages/admin/mail-server.component.html
  14. 2
      ui-ngx/src/app/modules/home/pages/admin/mail-server.component.ts
  15. 4
      ui-ngx/src/app/shared/models/query/query.models.ts
  16. 2
      ui-ngx/src/assets/locale/locale.constant-en_US.json

12
.travis.yml

@ -1,12 +0,0 @@
before_install:
- sudo rm -f /etc/mavenrc
- export M2_HOME=/usr/local/maven
- export MAVEN_OPTS="-Dmaven.repo.local=$HOME/.m2/repository -Xms1024m -Xmx3072m"
- export HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false
jdk:
- openjdk8
language: java
sudo: required
services:
- docker
script: mvn clean verify -Ddockerfile.skip=false

66
application/src/main/data/json/system/widget_bundles/maps.json

File diff suppressed because one or more lines are too long

2
application/src/main/data/upgrade/3.0.1/schema_update_to_uuid.sql

@ -86,6 +86,7 @@ BEGIN
DROP INDEX IF EXISTS idx_asset_customer_id; DROP INDEX IF EXISTS idx_asset_customer_id;
DROP INDEX IF EXISTS idx_asset_customer_id_and_type; DROP INDEX IF EXISTS idx_asset_customer_id_and_type;
DROP INDEX IF EXISTS idx_asset_type; DROP INDEX IF EXISTS idx_asset_type;
DROP INDEX IF EXISTS idx_attribute_kv_by_key_and_last_update_ts;
END; END;
$$; $$;
@ -105,6 +106,7 @@ BEGIN
CREATE INDEX IF NOT EXISTS idx_asset_customer_id ON asset(tenant_id, customer_id); CREATE INDEX IF NOT EXISTS idx_asset_customer_id ON asset(tenant_id, customer_id);
CREATE INDEX IF NOT EXISTS idx_asset_customer_id_and_type ON asset(tenant_id, customer_id, type); CREATE INDEX IF NOT EXISTS idx_asset_customer_id_and_type ON asset(tenant_id, customer_id, type);
CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type); CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type);
CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc);
END; END;
$$; $$;

131
dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java

@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityDataSortOrder;
import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.common.data.query.EntityFilter;
import org.thingsboard.server.common.data.query.EntityFilterType; import org.thingsboard.server.common.data.query.EntityFilterType;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.query.EntityListFilter;
import org.thingsboard.server.common.data.query.EntityNameFilter; import org.thingsboard.server.common.data.query.EntityNameFilter;
import org.thingsboard.server.common.data.query.EntitySearchQueryFilter; import org.thingsboard.server.common.data.query.EntitySearchQueryFilter;
@ -200,6 +201,9 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
entityTableMap.put(EntityType.TENANT, "tenant"); entityTableMap.put(EntityType.TENANT, "tenant");
} }
public static EntityType[] RELATION_QUERY_ENTITY_TYPES = new EntityType[]{
EntityType.TENANT, EntityType.CUSTOMER, EntityType.USER, EntityType.DASHBOARD, EntityType.ASSET, EntityType.DEVICE, EntityType.ENTITY_VIEW};
private static final String HIERARCHICAL_QUERY_TEMPLATE = " FROM (WITH RECURSIVE related_entities(from_id, from_type, to_id, to_type, relation_type, lvl) AS (" + private static final String HIERARCHICAL_QUERY_TEMPLATE = " FROM (WITH RECURSIVE related_entities(from_id, from_type, to_id, to_type, relation_type, lvl) AS (" +
" SELECT from_id, from_type, to_id, to_type, relation_type, 1 as lvl" + " SELECT from_id, from_type, to_id, to_type, relation_type, 1 as lvl" +
" FROM relation" + " FROM relation" +
@ -265,7 +269,8 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
String entityWhereClause = DefaultEntityQueryRepository.this.buildEntityWhere(ctx, query.getEntityFilter(), entityFieldsFiltersMapping); String entityWhereClause = DefaultEntityQueryRepository.this.buildEntityWhere(ctx, query.getEntityFilter(), entityFieldsFiltersMapping);
String latestJoins = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings); String latestJoinsCnt = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings, true);
String latestJoinsData = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings, false);
String whereClause = DefaultEntityQueryRepository.this.buildWhere(ctx, latestFiltersMapping, query.getEntityFilter().getType()); String whereClause = DefaultEntityQueryRepository.this.buildWhere(ctx, latestFiltersMapping, query.getEntityFilter().getType());
String textSearchQuery = DefaultEntityQueryRepository.this.buildTextSearchQuery(ctx, selectionMapping, pageLink.getTextSearch()); String textSearchQuery = DefaultEntityQueryRepository.this.buildTextSearchQuery(ctx, selectionMapping, pageLink.getTextSearch());
String entityFieldsSelection = EntityKeyMapping.buildSelections(entityFieldsSelectionMapping, query.getEntityFilter().getType(), entityType); String entityFieldsSelection = EntityKeyMapping.buildSelections(entityFieldsSelectionMapping, query.getEntityFilter().getType(), entityType);
@ -286,29 +291,44 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
topSelection = topSelection + ", " + latestSelection; topSelection = topSelection + ", " + latestSelection;
} }
String fromClause = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result %s", String fromClauseCount = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result %s",
"entities.*",
entityFieldsSelection,
addEntityTableQuery(ctx, query.getEntityFilter()),
entityWhereClause,
latestJoinsCnt,
whereClause,
textSearchQuery);
String fromClauseData = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result %s",
topSelection, topSelection,
entityFieldsSelection, entityFieldsSelection,
addEntityTableQuery(ctx, query.getEntityFilter()), addEntityTableQuery(ctx, query.getEntityFilter()),
entityWhereClause, entityWhereClause,
latestJoins, latestJoinsData,
whereClause, whereClause,
textSearchQuery); textSearchQuery);
int totalElements = jdbcTemplate.queryForObject(String.format("select count(*) %s", fromClause), ctx, Integer.class); if (!StringUtils.isEmpty(pageLink.getTextSearch())) {
//Unfortunately, we need to sacrifice performance in case of full text search, because it is applied to all joined records.
fromClauseCount = fromClauseData;
}
String countQuery = String.format("select count(id) %s", fromClauseCount);
int totalElements = jdbcTemplate.queryForObject(countQuery, ctx, Integer.class);
String dataQuery = String.format("select * %s", fromClause); String dataQuery = String.format("select * %s", fromClauseData);
EntityDataSortOrder sortOrder = pageLink.getSortOrder(); EntityDataSortOrder sortOrder = pageLink.getSortOrder();
if (sortOrder != null) { if (sortOrder != null) {
Optional<EntityKeyMapping> sortOrderMappingOpt = mappings.stream().filter(EntityKeyMapping::isSortOrder).findFirst(); Optional<EntityKeyMapping> sortOrderMappingOpt = mappings.stream().filter(EntityKeyMapping::isSortOrder).findFirst();
if (sortOrderMappingOpt.isPresent()) { if (sortOrderMappingOpt.isPresent()) {
EntityKeyMapping sortOrderMapping = sortOrderMappingOpt.get(); EntityKeyMapping sortOrderMapping = sortOrderMappingOpt.get();
dataQuery = String.format("%s order by %s", dataQuery, sortOrderMapping.getValueAlias()); String direction = sortOrder.getDirection() == EntityDataSortOrder.Direction.ASC ? "asc" : "desc";
if (sortOrder.getDirection() == EntityDataSortOrder.Direction.ASC) { if (sortOrderMapping.getEntityKey().getType() == EntityKeyType.ENTITY_FIELD) {
dataQuery += " asc"; dataQuery = String.format("%s order by %s %s", dataQuery, sortOrderMapping.getValueAlias(), direction);
} else { } else {
dataQuery += " desc"; dataQuery = String.format("%s order by %s %s, %s %s", dataQuery,
sortOrderMapping.getSortOrderNumAlias(), direction, sortOrderMapping.getSortOrderStrAlias(), direction);
} }
} }
} }
@ -329,10 +349,10 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
String entityFieldsQuery = EntityKeyMapping.buildQuery(ctx, entityFieldsFilters, entityFilter.getType()); String entityFieldsQuery = EntityKeyMapping.buildQuery(ctx, entityFieldsFilters, entityFilter.getType());
String result = permissionQuery; String result = permissionQuery;
if (!entityFilterQuery.isEmpty()) { if (!entityFilterQuery.isEmpty()) {
result += " and " + entityFilterQuery; result += " and (" + entityFilterQuery + ")";
} }
if (!entityFieldsQuery.isEmpty()) { if (!entityFieldsQuery.isEmpty()) {
result += " and " + entityFieldsQuery; result += " and (" + entityFieldsQuery + ")";
} }
return result; return result;
} }
@ -415,7 +435,12 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
String lvlFilter = getLvlFilter(entityFilter.getMaxLevel()); String lvlFilter = getLvlFilter(entityFilter.getMaxLevel());
String selectFields = "SELECT tenant_id, customer_id, id, created_time, type, name, label FROM " + entityType.name() + " WHERE id in ( SELECT entity_id"; String selectFields = "SELECT tenant_id, customer_id, id, created_time, type, name, label FROM " + entityType.name() + " WHERE id in ( SELECT entity_id";
String from = getQueryTemplate(entityFilter.getDirection()); String from = getQueryTemplate(entityFilter.getDirection());
String whereFilter = " WHERE re.relation_type = :where_relation_type AND re.to_type = :where_entity_type"; String whereFilter = " WHERE";
if (!StringUtils.isEmpty(entityFilter.getRelationType())) {
ctx.addStringParameter("where_relation_type", entityFilter.getRelationType());
whereFilter += " re.relation_type = :where_relation_type AND";
}
whereFilter += " re.to_type = :where_entity_type";
from = String.format(from, lvlFilter, whereFilter); from = String.format(from, lvlFilter, whereFilter);
String query = "( " + selectFields + from + ")"; String query = "( " + selectFields + from + ")";
@ -426,7 +451,6 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
query += " )"; query += " )";
ctx.addUuidParameter("relation_root_id", rootId.getId()); ctx.addUuidParameter("relation_root_id", rootId.getId());
ctx.addStringParameter("relation_root_type", rootId.getEntityType().name()); ctx.addStringParameter("relation_root_type", rootId.getEntityType().name());
ctx.addStringParameter("where_relation_type", entityFilter.getRelationType());
ctx.addStringParameter("where_entity_type", entityType.name()); ctx.addStringParameter("where_entity_type", entityType.name());
return query; return query;
} }
@ -446,46 +470,69 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
ctx.addUuidParameter("relation_root_id", rootId.getId()); ctx.addUuidParameter("relation_root_id", rootId.getId());
ctx.addStringParameter("relation_root_type", rootId.getEntityType().name()); ctx.addStringParameter("relation_root_type", rootId.getEntityType().name());
StringBuilder whereFilter; StringBuilder whereFilter = new StringBuilder();
;
boolean noConditions = true;
if (entityFilter.getFilters() != null && !entityFilter.getFilters().isEmpty()) { if (entityFilter.getFilters() != null && !entityFilter.getFilters().isEmpty()) {
whereFilter = new StringBuilder(" WHERE ");
boolean first = true;
boolean single = entityFilter.getFilters().size() == 1; boolean single = entityFilter.getFilters().size() == 1;
int entityTypeFilterIdx = 0; int entityTypeFilterIdx = 0;
for (EntityTypeFilter etf : entityFilter.getFilters()) { for (EntityTypeFilter etf : entityFilter.getFilters()) {
if (first) { String etfCondition = buildEtfCondition(ctx, etf, entityFilter.getDirection(), entityTypeFilterIdx++);
first = false; if (!etfCondition.isEmpty()) {
} else { if (noConditions) {
whereFilter.append(" AND "); whereFilter.append(" WHERE ");
} noConditions = false;
String relationType = etf.getRelationType(); } else {
if (!single) { whereFilter.append(" OR ");
whereFilter.append(" ("); }
} if (!single) {
List<String> whereEntityTypes = etf.getEntityTypes().stream().map(EntityType::name).collect(Collectors.toList()); whereFilter.append(" (");
whereFilter }
.append(" re.relation_type = :where_relation_type").append(entityTypeFilterIdx); whereFilter.append(etfCondition);
if (!whereEntityTypes.isEmpty()) { if (!single) {
whereFilter.append(" and re.") whereFilter.append(" )");
.append(entityFilter.getDirection().equals(EntitySearchDirection.FROM) ? "to" : "from") }
.append("_type in (:where_entity_types").append(entityTypeFilterIdx).append(")");
}
if (!single) {
whereFilter.append(" )");
}
ctx.addStringParameter("where_relation_type" + entityTypeFilterIdx, relationType);
if (!whereEntityTypes.isEmpty()) {
ctx.addStringListParameter("where_entity_types" + entityTypeFilterIdx, whereEntityTypes);
} }
entityTypeFilterIdx++;
} }
} else { }
whereFilter = new StringBuilder(); if (noConditions) {
whereFilter.append(" WHERE re.")
.append(entityFilter.getDirection().equals(EntitySearchDirection.FROM) ? "to" : "from")
.append("_type in (:where_entity_types").append(")");
ctx.addStringListParameter("where_entity_types", Arrays.stream(RELATION_QUERY_ENTITY_TYPES).map(EntityType::name).collect(Collectors.toList()));
} }
from = String.format(from, lvlFilter, whereFilter); from = String.format(from, lvlFilter, whereFilter);
return "( " + selectFields + from + ")"; return "( " + selectFields + from + ")";
} }
private String buildEtfCondition(QueryContext ctx, EntityTypeFilter etf, EntitySearchDirection direction, int entityTypeFilterIdx) {
StringBuilder whereFilter = new StringBuilder();
String relationType = etf.getRelationType();
List<EntityType> entityTypes = etf.getEntityTypes();
List<String> whereEntityTypes;
if (entityTypes == null || entityTypes.isEmpty()) {
whereEntityTypes = Collections.emptyList();
} else {
whereEntityTypes = etf.getEntityTypes().stream().map(EntityType::name).collect(Collectors.toList());
}
boolean hasRelationType = !StringUtils.isEmpty(relationType);
if (hasRelationType) {
ctx.addStringParameter("where_relation_type" + entityTypeFilterIdx, relationType);
whereFilter
.append("re.relation_type = :where_relation_type").append(entityTypeFilterIdx);
}
if (!whereEntityTypes.isEmpty()) {
if (hasRelationType) {
whereFilter.append(" and ");
}
whereFilter.append("re.")
.append(direction.equals(EntitySearchDirection.FROM) ? "to" : "from")
.append("_type in (:where_entity_types").append(entityTypeFilterIdx).append(")");
ctx.addStringListParameter("where_entity_types" + entityTypeFilterIdx, whereEntityTypes);
}
return whereFilter.toString();
}
private String getLvlFilter(int maxLevel) { private String getLvlFilter(int maxLevel) {
return maxLevel > 0 ? ("and lvl <= " + (maxLevel - 1)) : ""; return maxLevel > 0 ? ("and lvl <= " + (maxLevel - 1)) : "";
} }

74
dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java

@ -270,9 +270,10 @@ public class EntityKeyMapping {
Collectors.joining(", ")); Collectors.joining(", "));
} }
public static String buildLatestJoins(QueryContext ctx, EntityFilter entityFilter, EntityType entityType, List<EntityKeyMapping> latestMappings) { public static String buildLatestJoins(QueryContext ctx, EntityFilter entityFilter, EntityType entityType, List<EntityKeyMapping> latestMappings, boolean countQuery) {
return latestMappings.stream().map(mapping -> mapping.toLatestJoin(ctx, entityFilter, entityType)).collect( return latestMappings.stream().filter(mapping -> !countQuery || mapping.hasFilter())
Collectors.joining(" ")); .map(mapping -> mapping.toLatestJoin(ctx, entityFilter, entityType)).collect(
Collectors.joining(" "));
} }
public static String buildQuery(QueryContext ctx, List<EntityKeyMapping> mappings, EntityFilterType filterType) { public static String buildQuery(QueryContext ctx, List<EntityKeyMapping> mappings, EntityFilterType filterType) {
@ -363,19 +364,14 @@ public class EntityKeyMapping {
} }
private String buildAttributeSelection() { private String buildAttributeSelection() {
String attrValAlias = getValueAlias(); return buildTimeSeriesOrAttrSelection(true);
String attrTsAlias = getTsAlias();
String attrValSelection =
String.format("(coalesce(cast(%s.bool_v as varchar), '') || " +
"coalesce(%s.str_v, '') || " +
"coalesce(cast(%s.long_v as varchar), '') || " +
"coalesce(cast(%s.dbl_v as varchar), '') || " +
"coalesce(cast(%s.json_v as varchar), '')) as %s", alias, alias, alias, alias, alias, attrValAlias);
String attrTsSelection = String.format("%s.last_update_ts as %s", alias, attrTsAlias);
return String.join(", ", attrValSelection, attrTsSelection);
} }
private String buildTimeSeriesSelection() { private String buildTimeSeriesSelection() {
return buildTimeSeriesOrAttrSelection(false);
}
private String buildTimeSeriesOrAttrSelection(boolean attr) {
String attrValAlias = getValueAlias(); String attrValAlias = getValueAlias();
String attrTsAlias = getTsAlias(); String attrTsAlias = getTsAlias();
String attrValSelection = String attrValSelection =
@ -384,8 +380,25 @@ public class EntityKeyMapping {
"coalesce(cast(%s.long_v as varchar), '') || " + "coalesce(cast(%s.long_v as varchar), '') || " +
"coalesce(cast(%s.dbl_v as varchar), '') || " + "coalesce(cast(%s.dbl_v as varchar), '') || " +
"coalesce(cast(%s.json_v as varchar), '')) as %s", alias, alias, alias, alias, alias, attrValAlias); "coalesce(cast(%s.json_v as varchar), '')) as %s", alias, alias, alias, alias, alias, attrValAlias);
String attrTsSelection = String.format("%s.ts as %s", alias, attrTsAlias); String attrTsSelection = String.format("%s.%s as %s", alias, attr ? "last_update_ts" : "ts", attrTsAlias);
return String.join(", ", attrValSelection, attrTsSelection); if (this.isSortOrder) {
String attrNumAlias = getSortOrderNumAlias();
String attrVarcharAlias = getSortOrderStrAlias();
String attrSortOrderSelection =
String.format("coalesce(%s.dbl_v, cast(%s.long_v as double precision), (case when %s.bool_v then 1 else 0 end)) %s," +
"coalesce(%s.str_v, cast(%s.json_v as varchar), '') %s", alias, alias, alias, attrNumAlias, alias, alias, attrVarcharAlias);
return String.join(", ", attrValSelection, attrTsSelection, attrSortOrderSelection);
} else {
return String.join(", ", attrValSelection, attrTsSelection);
}
}
public String getSortOrderStrAlias() {
return getValueAlias() + "_so_varchar";
}
public String getSortOrderNumAlias() {
return getValueAlias() + "_so_num";
} }
private String buildKeyQuery(QueryContext ctx, String alias, KeyFilter keyFilter, private String buildKeyQuery(QueryContext ctx, String alias, KeyFilter keyFilter,
@ -425,7 +438,14 @@ public class EntityKeyMapping {
if (predicate.getType().equals(FilterPredicateType.NUMERIC)) { if (predicate.getType().equals(FilterPredicateType.NUMERIC)) {
return this.buildNumericPredicateQuery(ctx, field, (NumericFilterPredicate) predicate); return this.buildNumericPredicateQuery(ctx, field, (NumericFilterPredicate) predicate);
} else if (predicate.getType().equals(FilterPredicateType.STRING)) { } else if (predicate.getType().equals(FilterPredicateType.STRING)) {
return this.buildStringPredicateQuery(ctx, field, (StringFilterPredicate) predicate); if (key.getKey().equals("entityType") && !filterType.equals(EntityFilterType.RELATIONS_QUERY)) {
field = ctx.getEntityType().toString();
return this.buildStringPredicateQuery(ctx, field, (StringFilterPredicate) predicate)
.replace("lower(" + field, "lower('" + field + "'")
.replace(field + " ", "'" + field + "' ");
} else {
return this.buildStringPredicateQuery(ctx, field, (StringFilterPredicate) predicate);
}
} else { } else {
return this.buildBooleanPredicateQuery(ctx, field, (BooleanFilterPredicate) predicate); return this.buildBooleanPredicateQuery(ctx, field, (BooleanFilterPredicate) predicate);
} }
@ -460,30 +480,34 @@ public class EntityKeyMapping {
} }
switch (stringFilterPredicate.getOperation()) { switch (stringFilterPredicate.getOperation()) {
case EQUAL: case EQUAL:
stringOperationQuery = String.format("%s = :%s", operationField, paramName); stringOperationQuery = String.format("%s = :%s) or (%s is null and :%s = '')", operationField, paramName, operationField, paramName);
break; break;
case NOT_EQUAL: case NOT_EQUAL:
stringOperationQuery = String.format("%s != :%s", operationField, paramName); stringOperationQuery = String.format("%s != :%s) or (%s is null and :%s != '')", operationField, paramName, operationField, paramName);
break; break;
case STARTS_WITH: case STARTS_WITH:
value += "%"; value += "%";
stringOperationQuery = String.format("%s like :%s", operationField, paramName); stringOperationQuery = String.format("%s like :%s) or (%s is null and :%s = '%%')", operationField, paramName, operationField, paramName);
break; break;
case ENDS_WITH: case ENDS_WITH:
value = "%" + value; value = "%" + value;
stringOperationQuery = String.format("%s like :%s", operationField, paramName); stringOperationQuery = String.format("%s like :%s) or (%s is null and :%s = '%%')", operationField, paramName, operationField, paramName);
break; break;
case CONTAINS: case CONTAINS:
value = "%" + value + "%"; if (value.length() > 0) {
stringOperationQuery = String.format("%s like :%s", operationField, paramName); value = "%" + value + "%";
}
stringOperationQuery = String.format("%s like :%s) or (%s is null and :%s = '')", operationField, paramName, operationField, paramName);
break; break;
case NOT_CONTAINS: case NOT_CONTAINS:
value = "%" + value + "%"; if (value.length() > 0) {
stringOperationQuery = String.format("%s not like :%s", operationField, paramName); value = "%" + value + "%";
}
stringOperationQuery = String.format("%s not like :%s) or (%s is null and :%s != '')", operationField, paramName, operationField, paramName);
break; break;
} }
ctx.addStringParameter(paramName, value); ctx.addStringParameter(paramName, value);
return String.format("(%s is not null and %s)", field, stringOperationQuery); return String.format("((%s is not null and %s)", field, stringOperationQuery);
} }
private String buildNumericPredicateQuery(QueryContext ctx, String field, NumericFilterPredicate numericFilterPredicate) { private String buildNumericPredicateQuery(QueryContext ctx, String field, NumericFilterPredicate numericFilterPredicate) {

10
dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java

@ -15,6 +15,7 @@
*/ */
package org.thingsboard.server.dao.sql.query; package org.thingsboard.server.dao.sql.query;
import lombok.extern.slf4j.Slf4j;
import org.hibernate.type.PostgresUUIDType; import org.hibernate.type.PostgresUUIDType;
import org.springframework.jdbc.core.namedparam.SqlParameterSource; import org.springframework.jdbc.core.namedparam.SqlParameterSource;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
@ -27,6 +28,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.UUID; import java.util.UUID;
@Slf4j
public class QueryContext implements SqlParameterSource { public class QueryContext implements SqlParameterSource {
private static final PostgresUUIDType UUID_TYPE = new PostgresUUIDType(); private static final PostgresUUIDType UUID_TYPE = new PostgresUUIDType();
@ -41,10 +43,14 @@ public class QueryContext implements SqlParameterSource {
} }
void addParameter(String name, Object value, int type, String typeName) { void addParameter(String name, Object value, int type, String typeName) {
Parameter existing = params.put(name, new Parameter(value, type, typeName)); Parameter newParam = new Parameter(value, type, typeName);
if (existing != null) { Parameter oldParam = params.put(name, newParam);
if (oldParam != null && oldParam.value != null && !oldParam.value.equals(newParam.value)) {
throw new RuntimeException("Parameter with name: " + name + " was already registered!"); throw new RuntimeException("Parameter with name: " + name + " was already registered!");
} }
if(value == null){
log.warn("[{}][{}][{}] Trying to set null value", getTenantId(), getCustomerId(), name);
}
} }
public void append(String s) { public void append(String s) {

4
dao/src/main/resources/sql/schema-entities-idx.sql

@ -36,4 +36,6 @@ CREATE INDEX IF NOT EXISTS idx_asset_customer_id ON asset(tenant_id, customer_id
CREATE INDEX IF NOT EXISTS idx_asset_customer_id_and_type ON asset(tenant_id, customer_id, type); CREATE INDEX IF NOT EXISTS idx_asset_customer_id_and_type ON asset(tenant_id, customer_id, type);
CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type); CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type);
CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc);

722
dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java

@ -15,12 +15,14 @@
*/ */
package org.thingsboard.server.dao.service; package org.thingsboard.server.dao.service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import org.junit.After; import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.DataConstants;
@ -28,42 +30,23 @@ import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.asset.AssetSearchQuery; import org.thingsboard.server.common.data.id.*;
import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.kv.*;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; import org.thingsboard.server.common.data.query.*;
import org.thingsboard.server.common.data.query.DeviceSearchQueryFilter;
import org.thingsboard.server.common.data.query.DeviceTypeFilter;
import org.thingsboard.server.common.data.query.EntityCountQuery;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataPageLink;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityDataSortOrder;
import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.EntityListFilter;
import org.thingsboard.server.common.data.query.FilterPredicateValue;
import org.thingsboard.server.common.data.query.KeyFilter;
import org.thingsboard.server.common.data.query.NumericFilterPredicate;
import org.thingsboard.server.common.data.query.RelationsQueryFilter;
import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.EntitySearchDirection;
import org.thingsboard.server.common.data.relation.EntityTypeFilter; import org.thingsboard.server.common.data.relation.EntityTypeFilter;
import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import java.util.ArrayList; import java.util.*;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -72,6 +55,9 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
@Autowired @Autowired
private AttributesService attributesService; private AttributesService attributesService;
@Autowired
private TimeseriesService timeseriesService;
private TenantId tenantId; private TenantId tenantId;
@Before @Before
@ -88,6 +74,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
tenantService.deleteTenant(tenantId); tenantService.deleteTenant(tenantId);
} }
@Test @Test
public void testCountEntitiesByQuery() throws InterruptedException { public void testCountEntitiesByQuery() throws InterruptedException {
List<Device> devices = new ArrayList<>(); List<Device> devices = new ArrayList<>();
@ -131,6 +118,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
Assert.assertEquals(0, count); Assert.assertEquals(0, count);
} }
@Test @Test
public void testCountHierarchicalEntitiesByQuery() throws InterruptedException { public void testCountHierarchicalEntitiesByQuery() throws InterruptedException {
List<Asset> assets = new ArrayList<>(); List<Asset> assets = new ArrayList<>();
@ -195,6 +183,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
Assert.assertEquals(0, count); Assert.assertEquals(0, count);
} }
@Test @Test
public void testHierarchicalFindEntityDataWithAttributesByQuery() throws ExecutionException, InterruptedException { public void testHierarchicalFindEntityDataWithAttributesByQuery() throws ExecutionException, InterruptedException {
List<Asset> assets = new ArrayList<>(); List<Asset> assets = new ArrayList<>();
@ -266,6 +255,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
deviceService.deleteDevicesByTenantId(tenantId); deviceService.deleteDevicesByTenantId(tenantId);
} }
@Test @Test
public void testHierarchicalFindDevicesWithAttributesByQuery() throws ExecutionException, InterruptedException { public void testHierarchicalFindDevicesWithAttributesByQuery() throws ExecutionException, InterruptedException {
List<Asset> assets = new ArrayList<>(); List<Asset> assets = new ArrayList<>();
@ -338,6 +328,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
deviceService.deleteDevicesByTenantId(tenantId); deviceService.deleteDevicesByTenantId(tenantId);
} }
@Test @Test
public void testHierarchicalFindAssetsWithAttributesByQuery() throws ExecutionException, InterruptedException { public void testHierarchicalFindAssetsWithAttributesByQuery() throws ExecutionException, InterruptedException {
List<Asset> assets = new ArrayList<>(); List<Asset> assets = new ArrayList<>();
@ -456,6 +447,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
} }
} }
@Test @Test
public void testSimpleFindEntityDataByQuery() throws InterruptedException { public void testSimpleFindEntityDataByQuery() throws InterruptedException {
List<Device> devices = new ArrayList<>(); List<Device> devices = new ArrayList<>();
@ -526,6 +518,8 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
@Test @Test
public void testFindEntityDataByQueryWithAttributes() throws ExecutionException, InterruptedException { public void testFindEntityDataByQueryWithAttributes() throws ExecutionException, InterruptedException {
List<EntityKeyType> attributesEntityTypes = new ArrayList<>(Arrays.asList(EntityKeyType.CLIENT_ATTRIBUTE, EntityKeyType.SHARED_ATTRIBUTE, EntityKeyType.SERVER_ATTRIBUTE));
List<Device> devices = new ArrayList<>(); List<Device> devices = new ArrayList<>();
List<Long> temperatures = new ArrayList<>(); List<Long> temperatures = new ArrayList<>();
List<Long> highTemperatures = new ArrayList<>(); List<Long> highTemperatures = new ArrayList<>();
@ -545,6 +539,108 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
} }
} }
List<ListenableFuture<List<Void>>> attributeFutures = new ArrayList<>();
for (int i = 0; i < devices.size(); i++) {
Device device = devices.get(i);
for (String currentScope : DataConstants.allScopes()) {
attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), currentScope));
}
}
Futures.successfulAsList(attributeFutures).get();
DeviceTypeFilter filter = new DeviceTypeFilter();
filter.setDeviceType("default");
filter.setDeviceNameFilter("");
EntityDataSortOrder sortOrder = new EntityDataSortOrder(
new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC
);
EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
List<EntityKey> entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"));
for (EntityKeyType currentAttributeKeyType : attributesEntityTypes) {
List<EntityKey> latestValues = Collections.singletonList(new EntityKey(currentAttributeKeyType, "temperature"));
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null);
PageData<EntityData> data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
List<EntityData> loadedEntities = new ArrayList<>(data.getData());
while (data.hasNext()) {
query = query.next();
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities.addAll(data.getData());
}
Assert.assertEquals(67, loadedEntities.size());
List<String> loadedTemperatures = new ArrayList<>();
for (Device device : devices) {
loadedTemperatures.add(loadedEntities.stream().filter(entityData -> entityData.getEntityId().equals(device.getId())).findFirst().orElse(null)
.getLatest().get(currentAttributeKeyType).get("temperature").getValue());
}
List<String> deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
Assert.assertEquals(deviceTemperatures, loadedTemperatures);
pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
KeyFilter highTemperatureFilter = createNumericKeyFilter("temperature", currentAttributeKeyType, NumericFilterPredicate.NumericOperation.GREATER, 45);
List<KeyFilter> keyFiltersHighTemperature = Collections.singletonList(highTemperatureFilter);
query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersHighTemperature);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = new ArrayList<>(data.getData());
while (data.hasNext()) {
query = query.next();
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities.addAll(data.getData());
}
Assert.assertEquals(highTemperatures.size(), loadedEntities.size());
List<String> loadedHighTemperatures = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(currentAttributeKeyType).get("temperature").getValue()).collect(Collectors.toList());
List<String> deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures);
}
deviceService.deleteDevicesByTenantId(tenantId);
}
@Test
public void testBuildNumericPredicateQueryOperations() throws ExecutionException, InterruptedException{
List<Device> devices = new ArrayList<>();
List<Long> temperatures = new ArrayList<>();
List<Long> equalTemperatures = new ArrayList<>();
List<Long> notEqualTemperatures = new ArrayList<>();
List<Long> greaterTemperatures = new ArrayList<>();
List<Long> greaterOrEqualTemperatures = new ArrayList<>();
List<Long> lessTemperatures = new ArrayList<>();
List<Long> lessOrEqualTemperatures = new ArrayList<>();
for (int i = 0; i < 10; i++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("Device" + i);
device.setType("default");
device.setLabel("testLabel" + (int) (Math.random() * 1000));
devices.add(deviceService.saveDevice(device));
//TO make sure devices have different created time
Thread.sleep(1);
long temperature = (long) (Math.random() * 100);
temperatures.add(temperature);
if (temperature == 45) {
greaterOrEqualTemperatures.add(temperature);
lessOrEqualTemperatures.add(temperature);
equalTemperatures.add(temperature);
} else if (temperature > 45) {
greaterTemperatures.add(temperature);
greaterOrEqualTemperatures.add(temperature);
notEqualTemperatures.add(temperature);
} else {
lessTemperatures.add(temperature);
lessOrEqualTemperatures.add(temperature);
notEqualTemperatures.add(temperature);
}
}
List<ListenableFuture<List<Void>>> attributeFutures = new ArrayList<>(); List<ListenableFuture<List<Void>>> attributeFutures = new ArrayList<>();
for (int i = 0; i < devices.size(); i++) { for (int i = 0; i < devices.size(); i++) {
Device device = devices.get(i); Device device = devices.get(i);
@ -556,12 +652,158 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
filter.setDeviceType("default"); filter.setDeviceType("default");
filter.setDeviceNameFilter(""); filter.setDeviceNameFilter("");
EntityDataSortOrder sortOrder = new EntityDataSortOrder(
new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC
);
List<EntityKey> entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"));
List<EntityKey> latestValues = Collections.singletonList(new EntityKey(EntityKeyType.CLIENT_ATTRIBUTE, "temperature"));
KeyFilter greaterTemperatureFilter = createNumericKeyFilter("temperature", EntityKeyType.CLIENT_ATTRIBUTE, NumericFilterPredicate.NumericOperation.GREATER, 45);
List<KeyFilter> keyFiltersGreaterTemperature = Collections.singletonList(greaterTemperatureFilter);
KeyFilter greaterOrEqualTemperatureFilter = createNumericKeyFilter("temperature", EntityKeyType.CLIENT_ATTRIBUTE, NumericFilterPredicate.NumericOperation.GREATER_OR_EQUAL, 45);
List<KeyFilter> keyFiltersGreaterOrEqualTemperature = Collections.singletonList(greaterOrEqualTemperatureFilter);
KeyFilter lessTemperatureFilter = createNumericKeyFilter("temperature", EntityKeyType.CLIENT_ATTRIBUTE, NumericFilterPredicate.NumericOperation.LESS, 45);
List<KeyFilter> keyFiltersLessTemperature = Collections.singletonList(lessTemperatureFilter);
KeyFilter lessOrEqualTemperatureFilter = createNumericKeyFilter("temperature", EntityKeyType.CLIENT_ATTRIBUTE, NumericFilterPredicate.NumericOperation.LESS_OR_EQUAL, 45);
List<KeyFilter> keyFiltersLessOrEqualTemperature = Collections.singletonList(lessOrEqualTemperatureFilter);
KeyFilter equalTemperatureFilter = createNumericKeyFilter("temperature", EntityKeyType.CLIENT_ATTRIBUTE, NumericFilterPredicate.NumericOperation.EQUAL, 45);
List<KeyFilter> keyFiltersEqualTemperature = Collections.singletonList(equalTemperatureFilter);
KeyFilter notEqualTemperatureFilter = createNumericKeyFilter("temperature", EntityKeyType.CLIENT_ATTRIBUTE, NumericFilterPredicate.NumericOperation.NOT_EQUAL, 45);
List<KeyFilter> keyFiltersNotEqualTemperature = Collections.singletonList(notEqualTemperatureFilter);
//Greater Operation
EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersGreaterTemperature);
PageData<EntityData> data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
List<EntityData> loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(greaterTemperatures.size(), loadedEntities.size());
List<String> loadedTemperatures = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList());
List<String> deviceTemperatures = greaterTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
Assert.assertEquals(deviceTemperatures, loadedTemperatures);
//Greater or equal Operation
pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersGreaterOrEqualTemperature);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(greaterOrEqualTemperatures.size(), loadedEntities.size());
loadedTemperatures = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList());
deviceTemperatures = greaterOrEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
Assert.assertEquals(deviceTemperatures, loadedTemperatures);
//Less Operation
pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersLessTemperature);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(lessTemperatures.size(), loadedEntities.size());
loadedTemperatures = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList());
deviceTemperatures = lessTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
Assert.assertEquals(deviceTemperatures, loadedTemperatures);
//Less or equal Operation
pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersLessOrEqualTemperature);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(lessOrEqualTemperatures.size(), loadedEntities.size());
loadedTemperatures = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList());
deviceTemperatures = lessOrEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
Assert.assertEquals(deviceTemperatures, loadedTemperatures);
//Equal Operation
pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersEqualTemperature);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(equalTemperatures.size(), loadedEntities.size());
loadedTemperatures = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList());
deviceTemperatures = equalTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
Assert.assertEquals(deviceTemperatures, loadedTemperatures);
//Not equal Operation
pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersNotEqualTemperature);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(notEqualTemperatures.size(), loadedEntities.size());
loadedTemperatures = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList());
deviceTemperatures = notEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
Assert.assertEquals(deviceTemperatures, loadedTemperatures);
deviceService.deleteDevicesByTenantId(tenantId);
}
@Test
public void testFindEntityDataByQueryWithTimeseries() throws ExecutionException, InterruptedException {
List<Device> devices = new ArrayList<>();
List<Double> temperatures = new ArrayList<>();
List<Double> highTemperatures = new ArrayList<>();
for (int i = 0; i < 67; i++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("Device" + i);
device.setType("default");
device.setLabel("testLabel" + (int) (Math.random() * 1000));
devices.add(deviceService.saveDevice(device));
//TO make sure devices have different created time
Thread.sleep(1);
double temperature = (double) (Math.random() * 100.0);
temperatures.add(temperature);
if (temperature > 45.0) {
highTemperatures.add(temperature);
}
}
List<ListenableFuture<List<Void>>> timeseriesFutures = new ArrayList<>();
for (int i = 0; i < devices.size(); i++) {
Device device = devices.get(i);
timeseriesFutures.add(saveLongTimeseries(device.getId(), "temperature", temperatures.get(i)));
}
Futures.successfulAsList(timeseriesFutures).get();
DeviceTypeFilter filter = new DeviceTypeFilter();
filter.setDeviceType("default");
filter.setDeviceNameFilter("");
EntityDataSortOrder sortOrder = new EntityDataSortOrder( EntityDataSortOrder sortOrder = new EntityDataSortOrder(
new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC
); );
EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder); EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
List<EntityKey> entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); List<EntityKey> entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"));
List<EntityKey> latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); List<EntityKey> latestValues = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "temperature"));
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null);
PageData<EntityData> data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); PageData<EntityData> data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
@ -576,14 +818,14 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
List<String> loadedTemperatures = new ArrayList<>(); List<String> loadedTemperatures = new ArrayList<>();
for (Device device : devices) { for (Device device : devices) {
loadedTemperatures.add(loadedEntities.stream().filter(entityData -> entityData.getEntityId().equals(device.getId())).findFirst().orElse(null) loadedTemperatures.add(loadedEntities.stream().filter(entityData -> entityData.getEntityId().equals(device.getId())).findFirst().orElse(null)
.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()); .getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue());
} }
List<String> deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); List<String> deviceTemperatures = temperatures.stream().map(aDouble -> Double.toString(aDouble)).collect(Collectors.toList());
Assert.assertEquals(deviceTemperatures, loadedTemperatures); Assert.assertEquals(deviceTemperatures, loadedTemperatures);
pageLink = new EntityDataPageLink(10, 0, null, sortOrder); pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
KeyFilter highTemperatureFilter = new KeyFilter(); KeyFilter highTemperatureFilter = new KeyFilter();
highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); highTemperatureFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature"));
NumericFilterPredicate predicate = new NumericFilterPredicate(); NumericFilterPredicate predicate = new NumericFilterPredicate();
predicate.setValue(FilterPredicateValue.fromDouble(45)); predicate.setValue(FilterPredicateValue.fromDouble(45));
predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);
@ -603,17 +845,425 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); Assert.assertEquals(highTemperatures.size(), loadedEntities.size());
List<String> loadedHighTemperatures = loadedEntities.stream().map(entityData -> List<String> loadedHighTemperatures = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); entityData.getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()).collect(Collectors.toList());
List<String> deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); List<String> deviceHighTemperatures = highTemperatures.stream().map(aDouble -> Double.toString(aDouble)).collect(Collectors.toList());
Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures);
deviceService.deleteDevicesByTenantId(tenantId); deviceService.deleteDevicesByTenantId(tenantId);
} }
@Test
public void testBuildStringPredicateQueryOperations() throws ExecutionException, InterruptedException{
List<Device> devices = new ArrayList<>();
List<String> attributeStrings = new ArrayList<>();
List<String> equalStrings = new ArrayList<>();
List<String> notEqualStrings = new ArrayList<>();
List<String> startsWithStrings = new ArrayList<>();
List<String> endsWithStrings = new ArrayList<>();
List<String> containsStrings = new ArrayList<>();
List<String> notContainsStrings = new ArrayList<>();
for (int i = 0; i < 10; i++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("Device" + i);
device.setType("default");
device.setLabel("testLabel" + (int) (Math.random() * 1000));
devices.add(deviceService.saveDevice(device));
//TO make sure devices have different created time
Thread.sleep(1);
List<StringFilterPredicate.StringOperation> operationValues= Arrays.asList(StringFilterPredicate.StringOperation.values());
StringFilterPredicate.StringOperation operation = operationValues.get(new Random().nextInt(operationValues.size()));
String operationName = operation.name();
attributeStrings.add(operationName);
switch(operation){
case EQUAL:
equalStrings.add(operationName);
notContainsStrings.add(operationName);
notEqualStrings.add(operationName);
break;
case NOT_EQUAL:
notContainsStrings.add(operationName);
break;
case STARTS_WITH:
notEqualStrings.add(operationName);
startsWithStrings.add(operationName);
endsWithStrings.add(operationName);
notContainsStrings.add(operationName);
break;
case ENDS_WITH:
notEqualStrings.add(operationName);
endsWithStrings.add(operationName);
notContainsStrings.add(operationName);
break;
case CONTAINS:
notEqualStrings.add(operationName);
notContainsStrings.add(operationName);
containsStrings.add(operationName);
break;
case NOT_CONTAINS:
notEqualStrings.add(operationName);
containsStrings.add(operationName);
break;
}
}
List<ListenableFuture<List<Void>>> attributeFutures = new ArrayList<>();
for (int i = 0; i < devices.size(); i++) {
Device device = devices.get(i);
attributeFutures.add(saveStringAttribute(device.getId(), "attributeString", attributeStrings.get(i), DataConstants.CLIENT_SCOPE));
}
Futures.successfulAsList(attributeFutures).get();
DeviceTypeFilter filter = new DeviceTypeFilter();
filter.setDeviceType("default");
filter.setDeviceNameFilter("");
EntityDataSortOrder sortOrder = new EntityDataSortOrder(
new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC
);
List<EntityKey> entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"),
new EntityKey(EntityKeyType.ENTITY_FIELD, "entityType"));
List<EntityKey> latestValues = Collections.singletonList(new EntityKey(EntityKeyType.CLIENT_ATTRIBUTE, "attributeString"));
List<KeyFilter> keyFiltersEqualString = createStringKeyFilters("attributeString", EntityKeyType.CLIENT_ATTRIBUTE, StringFilterPredicate.StringOperation.EQUAL, "equal");
List<KeyFilter> keyFiltersNotEqualString = createStringKeyFilters("attributeString", EntityKeyType.CLIENT_ATTRIBUTE, StringFilterPredicate.StringOperation.NOT_EQUAL, "NOT_EQUAL");
List<KeyFilter> keyFiltersStartsWithString = createStringKeyFilters("attributeString", EntityKeyType.CLIENT_ATTRIBUTE, StringFilterPredicate.StringOperation.STARTS_WITH, "starts_");
List<KeyFilter> keyFiltersEndsWithString = createStringKeyFilters("attributeString", EntityKeyType.CLIENT_ATTRIBUTE, StringFilterPredicate.StringOperation.ENDS_WITH, "_WITH");
List<KeyFilter> keyFiltersContainsString = createStringKeyFilters("attributeString", EntityKeyType.CLIENT_ATTRIBUTE, StringFilterPredicate.StringOperation.CONTAINS, "contains");
List<KeyFilter> keyFiltersNotContainsString = createStringKeyFilters("attributeString", EntityKeyType.CLIENT_ATTRIBUTE, StringFilterPredicate.StringOperation.NOT_CONTAINS, "NOT_CONTAINS");
List<KeyFilter> deviceTypeFilters = createStringKeyFilters("entityType", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.NOT_EQUAL, "NOT_EQUAL");
// Equal Operation
EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersEqualString);
PageData<EntityData> data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
List<EntityData> loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(equalStrings.size(), loadedEntities.size());
List<String> loadedStrings = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("attributeString").getValue()).collect(Collectors.toList());
Assert.assertTrue(listEqualWithoutOrder(equalStrings, loadedStrings));
// Not equal Operation
pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersNotEqualString);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(notEqualStrings.size(), loadedEntities.size());
loadedStrings = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("attributeString").getValue()).collect(Collectors.toList());
Assert.assertTrue(listEqualWithoutOrder(notEqualStrings, loadedStrings));
// Starts with Operation
pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersStartsWithString);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(startsWithStrings.size(), loadedEntities.size());
loadedStrings = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("attributeString").getValue()).collect(Collectors.toList());
Assert.assertTrue(listEqualWithoutOrder(startsWithStrings, loadedStrings));
// Ends with Operation
pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersEndsWithString);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(endsWithStrings.size(), loadedEntities.size());
loadedStrings = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("attributeString").getValue()).collect(Collectors.toList());
Assert.assertTrue(listEqualWithoutOrder(endsWithStrings, loadedStrings));
// Contains Operation
pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersContainsString);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(containsStrings.size(), loadedEntities.size());
loadedStrings = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("attributeString").getValue()).collect(Collectors.toList());
Assert.assertTrue(listEqualWithoutOrder(containsStrings, loadedStrings));
// Not contains Operation
pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersNotContainsString);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(notContainsStrings.size(), loadedEntities.size());
loadedStrings = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("attributeString").getValue()).collect(Collectors.toList());
Assert.assertTrue(listEqualWithoutOrder(notContainsStrings, loadedStrings));
// Device type filters Operation
pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, deviceTypeFilters);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(devices.size(), loadedEntities.size());
deviceService.deleteDevicesByTenantId(tenantId);
}
@Test
public void testBuildStringPredicateQueryOperationsForEntityType() throws ExecutionException, InterruptedException{
List<Device> devices = new ArrayList<>();
for (int i = 0; i < 10; i++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("Device" + i);
device.setType("default");
device.setLabel("testLabel" + (int) (Math.random() * 1000));
devices.add(deviceService.saveDevice(device));
//TO make sure devices have different created time
Thread.sleep(1);
}
DeviceTypeFilter filter = new DeviceTypeFilter();
filter.setDeviceType("default");
filter.setDeviceNameFilter("");
EntityDataSortOrder sortOrder = new EntityDataSortOrder(
new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC
);
List<EntityKey> entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"),
new EntityKey(EntityKeyType.ENTITY_FIELD, "entityType"));
List<KeyFilter> keyFiltersEqualString = createStringKeyFilters("entityType", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.EQUAL, "device");
List<KeyFilter> keyFiltersNotEqualString = createStringKeyFilters("entityType", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.NOT_EQUAL, "asset");
List<KeyFilter> keyFiltersStartsWithString = createStringKeyFilters("entityType", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "dev");
List<KeyFilter> keyFiltersEndsWithString = createStringKeyFilters("entityType", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.ENDS_WITH, "ice");
List<KeyFilter> keyFiltersContainsString = createStringKeyFilters("entityType", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.CONTAINS, "vic");
List<KeyFilter> keyFiltersNotContainsString = createStringKeyFilters("entityType", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.NOT_CONTAINS, "dolphin");
// Equal Operation
EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersEqualString);
PageData<EntityData> data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
List<EntityData> loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(devices.size(), loadedEntities.size());
List<String> loadedStrings = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).collect(Collectors.toList());
List<String> devicesNames = devices.stream().map(Device::getName).collect(Collectors.toList());
Assert.assertTrue(listEqualWithoutOrder(devicesNames, loadedStrings));
// Not equal Operation
pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersNotEqualString);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(devices.size(), loadedEntities.size());
loadedStrings = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).collect(Collectors.toList());
Assert.assertTrue(listEqualWithoutOrder(devicesNames, loadedStrings));
// Starts with Operation
pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersStartsWithString);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(devices.size(), loadedEntities.size());
loadedStrings = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).collect(Collectors.toList());
Assert.assertTrue(listEqualWithoutOrder(devicesNames, loadedStrings));
// Ends with Operation
pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersEndsWithString);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(devices.size(), loadedEntities.size());
loadedStrings = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).collect(Collectors.toList());
Assert.assertTrue(listEqualWithoutOrder(devicesNames, loadedStrings));
// Contains Operation
pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersContainsString);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(devices.size(), loadedEntities.size());
loadedStrings = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).collect(Collectors.toList());
Assert.assertTrue(listEqualWithoutOrder(devicesNames, loadedStrings));
// Not contains Operation
pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersNotContainsString);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(devices.size(), loadedEntities.size());
loadedStrings = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).collect(Collectors.toList());
Assert.assertTrue(listEqualWithoutOrder(devicesNames, loadedStrings));
deviceService.deleteDevicesByTenantId(tenantId);
}
@Test
public void testBuildSimplePredicateQueryOperations() throws InterruptedException{
List<Device> devices = new ArrayList<>();
for (int i = 0; i < 10; i++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("Device" + i);
device.setType("default");
device.setLabel("testLabel" + (int) (Math.random() * 1000));
devices.add(deviceService.saveDevice(device));
//TO make sure devices have different created time
Thread.sleep(1);
}
DeviceTypeFilter filter = new DeviceTypeFilter();
filter.setDeviceType("default");
filter.setDeviceNameFilter("");
EntityDataSortOrder sortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), EntityDataSortOrder.Direction.DESC);
List<KeyFilter> deviceTypeFilters = createStringKeyFilters("type", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.EQUAL, "default");
KeyFilter createdTimeFilter = createNumericKeyFilter("createdTime", EntityKeyType.ENTITY_FIELD, NumericFilterPredicate.NumericOperation.GREATER, 1L);
List<KeyFilter> createdTimeFilters = Collections.singletonList(createdTimeFilter);
List<KeyFilter> nameFilters = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.CONTAINS, "Device");
List<EntityKey> entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"),
new EntityKey(EntityKeyType.ENTITY_FIELD, "type"));
// Device type filters
EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, deviceTypeFilters);
PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
List<EntityData> loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(devices.size(), loadedEntities.size());
// Device create time filters
pageLink = new EntityDataPageLink(100, 0, null, sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, null, createdTimeFilters);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(devices.size(), loadedEntities.size());
// Device name filters
pageLink = new EntityDataPageLink(100, 0, null, null);
query = new EntityDataQuery(filter, pageLink, entityFields, null, nameFilters);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = getLoadedEntities(data, query);
Assert.assertEquals(devices.size(), loadedEntities.size());
deviceService.deleteDevicesByTenantId(tenantId);
}
private Boolean listEqualWithoutOrder(List<String> A, List<String> B) {
return A.containsAll(B) && B.containsAll(A);
}
private List<EntityData> getLoadedEntities(PageData data, EntityDataQuery query) {
List<EntityData> loadedEntities = new ArrayList<>(data.getData());
while (data.hasNext()) {
query = query.next();
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities.addAll(data.getData());
}
return loadedEntities;
}
private List<KeyFilter> createStringKeyFilters(String key, EntityKeyType keyType, StringFilterPredicate.StringOperation operation, String value){
KeyFilter filter = new KeyFilter();
filter.setKey(new EntityKey(keyType, key));
StringFilterPredicate predicate = new StringFilterPredicate();
predicate.setValue(FilterPredicateValue.fromString(value));
predicate.setOperation(operation);
predicate.setIgnoreCase(true);
filter.setPredicate(predicate);
return Collections.singletonList(filter);
}
private KeyFilter createNumericKeyFilter(String key, EntityKeyType keyType, NumericFilterPredicate.NumericOperation operation, double value){
KeyFilter filter = new KeyFilter();
filter.setKey(new EntityKey(keyType, key));
NumericFilterPredicate predicate = new NumericFilterPredicate();
predicate.setValue(FilterPredicateValue.fromDouble(value));
predicate.setOperation(operation);
filter.setPredicate(predicate);
return filter;
}
private ListenableFuture<List<Void>> saveLongAttribute(EntityId entityId, String key, long value, String scope) { private ListenableFuture<List<Void>> saveLongAttribute(EntityId entityId, String key, long value, String scope) {
KvEntry attrValue = new LongDataEntry(key, value); KvEntry attrValue = new LongDataEntry(key, value);
AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L);
return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr));
} }
private ListenableFuture<List<Void>> saveStringAttribute(EntityId entityId, String key, String value, String scope) {
KvEntry attrValue = new StringDataEntry(key, value);
AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L);
return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr));
}
private ListenableFuture<List<Void>> saveLongTimeseries(EntityId entityId, String key, Double value) {
TsKvEntity tsKv = new TsKvEntity();
tsKv.setStrKey(key);
tsKv.setDoubleValue(value);
KvEntry telemetryValue = new DoubleDataEntry(key, value);
BasicTsKvEntry timeseries = new BasicTsKvEntry(42L, telemetryValue);
return timeseriesService.save(SYSTEM_TENANT_ID, entityId, timeseries);
}
} }

2
ui-ngx/src/app/modules/home/components/attribute/add-attribute-dialog.component.ts

@ -75,7 +75,7 @@ export class AddAttributeDialogComponent extends DialogComponent<AddAttributeDia
this.submitted = true; this.submitted = true;
const attribute: AttributeData = { const attribute: AttributeData = {
lastUpdateTs: null, lastUpdateTs: null,
key: this.attributeFormGroup.get('key').value, key: this.attributeFormGroup.get('key').value.trim(),
value: this.attributeFormGroup.get('value').value value: this.attributeFormGroup.get('value').value
}; };
this.attributeService.saveEntityAttributes(this.data.entityId, this.attributeService.saveEntityAttributes(this.data.entityId,

2
ui-ngx/src/app/modules/home/components/entity/entity.component.ts

@ -129,7 +129,7 @@ export abstract class EntityComponent<T extends BaseData<HasId>,
acc[curr] = obj[curr]; acc[curr] = obj[curr];
} }
return acc; return acc;
}, {}); }, Array.isArray(obj) ? [] : {});
} }
protected setEntitiesTableConfig(entitiesTableConfig: C) { protected setEntitiesTableConfig(entitiesTableConfig: C) {

23
ui-ngx/src/app/modules/home/components/filter/key-filter-dialog.component.html

@ -29,13 +29,6 @@
<fieldset [disabled]="isLoading$ | async" fxLayout="column"> <fieldset [disabled]="isLoading$ | async" fxLayout="column">
<section fxLayout="row" fxLayoutGap="8px" class="entity-key"> <section fxLayout="row" fxLayoutGap="8px" class="entity-key">
<section fxFlex="70" fxLayout="row" formGroupName="key" fxLayoutGap="8px"> <section fxFlex="70" fxLayout="row" formGroupName="key" fxLayoutGap="8px">
<mat-form-field fxFlex="60" class="mat-block">
<mat-label translate>filter.key-name</mat-label>
<input matInput required formControlName="key">
<mat-error *ngIf="keyFilterFormGroup.get('key.key').hasError('required')">
{{ 'filter.key-name-required' | translate }}
</mat-error>
</mat-form-field>
<mat-form-field fxFlex="40" class="mat-block"> <mat-form-field fxFlex="40" class="mat-block">
<mat-label translate>filter.key-type.key-type</mat-label> <mat-label translate>filter.key-type.key-type</mat-label>
<mat-select required formControlName="type"> <mat-select required formControlName="type">
@ -44,10 +37,24 @@
</mat-option> </mat-option>
</mat-select> </mat-select>
</mat-form-field> </mat-form-field>
<mat-form-field fxFlex="60" class="mat-block">
<mat-label translate>filter.key-name</mat-label>
<input matInput required formControlName="key"
[matAutocomplete]="auto"
[matAutocompleteDisabled]="keyFilterFormGroup.get('key.type').value !== entityField">
<mat-autocomplete autoActiveFirstOption #auto="matAutocomplete">
<mat-option *ngFor="let option of filteredEntityFields | async" [value]="option">
{{option}}
</mat-option>
</mat-autocomplete>
<mat-error *ngIf="keyFilterFormGroup.get('key.key').hasError('required')">
{{ 'filter.key-name-required' | translate }}
</mat-error>
</mat-form-field>
</section> </section>
<mat-form-field fxFlex="30" class="mat-block"> <mat-form-field fxFlex="30" class="mat-block">
<mat-label translate>filter.value-type.value-type</mat-label> <mat-label translate>filter.value-type.value-type</mat-label>
<mat-select matInput formControlName="valueType"> <mat-select formControlName="valueType">
<mat-select-trigger> <mat-select-trigger>
<mat-icon class="tb-mat-18" svgIcon="{{ entityKeyValueTypes.get(keyFilterFormGroup.get('valueType').value)?.icon }}"></mat-icon> <mat-icon class="tb-mat-18" svgIcon="{{ entityKeyValueTypes.get(keyFilterFormGroup.get('valueType').value)?.icon }}"></mat-icon>
<span>{{ entityKeyValueTypes.get(keyFilterFormGroup.get('valueType').value)?.name | translate }}</span> <span>{{ entityKeyValueTypes.get(keyFilterFormGroup.get('valueType').value)?.name | translate }}</span>

33
ui-ngx/src/app/modules/home/components/filter/key-filter-dialog.component.ts

@ -27,10 +27,14 @@ import {
entityKeyTypeTranslationMap, entityKeyTypeTranslationMap,
EntityKeyValueType, EntityKeyValueType,
entityKeyValueTypesMap, entityKeyValueTypesMap,
KeyFilterInfo, KeyFilterPredicate KeyFilterInfo,
KeyFilterPredicate
} from '@shared/models/query/query.models'; } from '@shared/models/query/query.models';
import { DialogService } from '@core/services/dialog.service'; import { DialogService } from '@core/services/dialog.service';
import { TranslateService } from '@ngx-translate/core'; import { TranslateService } from '@ngx-translate/core';
import { EntityField, entityFields } from '@shared/models/entity.models';
import { Observable } from 'rxjs';
import { filter, map, startWith } from 'rxjs/operators';
export interface KeyFilterDialogData { export interface KeyFilterDialogData {
keyFilter: KeyFilterInfo; keyFilter: KeyFilterInfo;
@ -61,6 +65,14 @@ export class KeyFilterDialogComponent extends
submitted = false; submitted = false;
entityFields: { [fieldName: string]: EntityField };
entityFieldsList: string[];
readonly entityField = EntityKeyType.ENTITY_FIELD;
filteredEntityFields: Observable<string[]>;
constructor(protected store: Store<AppState>, constructor(protected store: Store<AppState>,
protected router: Router, protected router: Router,
@Inject(MAT_DIALOG_DATA) public data: KeyFilterDialogData, @Inject(MAT_DIALOG_DATA) public data: KeyFilterDialogData,
@ -99,9 +111,28 @@ export class KeyFilterDialogComponent extends
); );
} }
}); });
this.keyFilterFormGroup.get('key.key').valueChanges.pipe(
filter((keyName) => this.keyFilterFormGroup.get('key.type').value === this.entityField && this.entityFields.hasOwnProperty(keyName))
).subscribe((keyName: string) => {
const prevValueType: EntityKeyValueType = this.keyFilterFormGroup.value.valueType;
const newValueType = this.entityFields[keyName]?.time ? EntityKeyValueType.DATE_TIME : EntityKeyValueType.STRING;
if (prevValueType !== newValueType) {
this.keyFilterFormGroup.get('valueType').patchValue(newValueType, {emitEvent: false});
}
});
this.entityFields = entityFields;
this.entityFieldsList = Object.values(entityFields).map(entityField => entityField.keyName).sort();
} }
ngOnInit(): void { ngOnInit(): void {
this.filteredEntityFields = this.keyFilterFormGroup.get('key.key').valueChanges.pipe(
startWith(''),
map(value => {
return this.entityFieldsList.filter(option => option.startsWith(value));
})
);
} }
isErrorState(control: FormControl | null, form: FormGroupDirective | NgForm | null): boolean { isErrorState(control: FormControl | null, form: FormGroupDirective | NgForm | null): boolean {

8
ui-ngx/src/app/modules/home/pages/admin/mail-server.component.html

@ -118,16 +118,18 @@
</mat-form-field> </mat-form-field>
<mat-form-field class="mat-block"> <mat-form-field class="mat-block">
<mat-label translate>admin.proxy-password</mat-label> <mat-label translate>admin.proxy-password</mat-label>
<input matInput formControlName="proxyPassword"> <input matInput type="password" formControlName="proxyPassword" autocomplete="new-proxy-password">
</mat-form-field> </mat-form-field>
</div> </div>
<mat-form-field class="mat-block"> <mat-form-field class="mat-block">
<mat-label translate>common.username</mat-label> <mat-label translate>common.username</mat-label>
<input matInput formControlName="username" placeholder="{{ 'common.enter-username' | translate }}"/> <input matInput formControlName="username" placeholder="{{ 'common.enter-username' | translate }}"
autocomplete="new-username"/>
</mat-form-field> </mat-form-field>
<mat-form-field class="mat-block"> <mat-form-field class="mat-block">
<mat-label translate>common.password</mat-label> <mat-label translate>common.password</mat-label>
<input matInput formControlName="password" type="password" placeholder="{{ 'common.enter-password' | translate }}"/> <input matInput formControlName="password" type="password"
placeholder="{{ 'common.enter-password' | translate }}" autocomplete="new-password"/>
</mat-form-field> </mat-form-field>
<div fxLayout="row" fxLayoutAlign="end center" fxLayout.xs="column" fxLayoutAlign.xs="end" fxLayoutGap="16px"> <div fxLayout="row" fxLayoutAlign="end center" fxLayout.xs="column" fxLayoutAlign.xs="end" fxLayoutGap="16px">
<button mat-raised-button type="button" <button mat-raised-button type="button"

2
ui-ngx/src/app/modules/home/pages/admin/mail-server.component.ts

@ -92,7 +92,7 @@ export class MailServerComponent extends PageComponent implements OnInit, HasCon
} }
enableProxyChanged(): void { enableProxyChanged(): void {
let enableProxy: boolean = this.mailSettings.get('enableProxy').value; const enableProxy: boolean = this.mailSettings.get('enableProxy').value;
if (enableProxy) { if (enableProxy) {
this.mailSettings.get('proxyHost').enable(); this.mailSettings.get('proxyHost').enable();
this.mailSettings.get('proxyPort').enable(); this.mailSettings.get('proxyPort').enable();

4
ui-ngx/src/app/shared/models/query/query.models.ts

@ -203,7 +203,7 @@ export enum StringOperation {
STARTS_WITH = 'STARTS_WITH', STARTS_WITH = 'STARTS_WITH',
ENDS_WITH = 'ENDS_WITH', ENDS_WITH = 'ENDS_WITH',
CONTAINS = 'CONTAINS', CONTAINS = 'CONTAINS',
NOT_CONTAIN = 'NOT_CONTAIN' NOT_CONTAINS = 'NOT_CONTAINS'
} }
export const stringOperationTranslationMap = new Map<StringOperation, string>( export const stringOperationTranslationMap = new Map<StringOperation, string>(
@ -213,7 +213,7 @@ export const stringOperationTranslationMap = new Map<StringOperation, string>(
[StringOperation.STARTS_WITH, 'filter.operation.starts-with'], [StringOperation.STARTS_WITH, 'filter.operation.starts-with'],
[StringOperation.ENDS_WITH, 'filter.operation.ends-with'], [StringOperation.ENDS_WITH, 'filter.operation.ends-with'],
[StringOperation.CONTAINS, 'filter.operation.contains'], [StringOperation.CONTAINS, 'filter.operation.contains'],
[StringOperation.NOT_CONTAIN, 'filter.operation.not-contain'] [StringOperation.NOT_CONTAINS, 'filter.operation.not-contains']
] ]
); );

2
ui-ngx/src/assets/locale/locale.constant-en_US.json

@ -1189,7 +1189,7 @@
"starts-with": "starts with", "starts-with": "starts with",
"ends-with": "ends with", "ends-with": "ends with",
"contains": "contains", "contains": "contains",
"not-contain": "not contain", "not-contains": "not contains",
"greater": "greater than", "greater": "greater than",
"less": "less than", "less": "less than",
"greater-or-equal": "greater or equal", "greater-or-equal": "greater or equal",

Loading…
Cancel
Save