tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Fix for sort order in search by attributes or telemetry

pull/3174/head
Andrii Shvaika 6 years ago
parent
112b92503e
commit
d83d9ec3df
5 changed files with 66 additions and 31 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      application/src/main/data/upgrade/3.0.1/schema_update_to_uuid.sql
  2. 35
      dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java
  3. 51
      dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java
  4. 5
      dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java
  5. 4
      dao/src/main/resources/sql/schema-entities-idx.sql

2
application/src/main/data/upgrade/3.0.1/schema_update_to_uuid.sql
View File

@ -86,6 +86,7 @@ BEGIN
DROP INDEX IF EXISTS idx_asset_customer_id;
DROP INDEX IF EXISTS idx_asset_customer_id_and_type;
DROP INDEX IF EXISTS idx_asset_type;
DROP INDEX IF EXISTS idx_attribute_kv_by_key_and_last_update_ts;
END;
$$;
@ -105,6 +106,7 @@ BEGIN
CREATE INDEX IF NOT EXISTS idx_asset_customer_id ON asset(tenant_id, customer_id);
CREATE INDEX IF NOT EXISTS idx_asset_customer_id_and_type ON asset(tenant_id, customer_id, type);
CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type);
CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc);
END;
$$;

35
dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java
View File

@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityDataSortOrder;
import org.thingsboard.server.common.data.query.EntityFilter;
import org.thingsboard.server.common.data.query.EntityFilterType;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.EntityListFilter;
import org.thingsboard.server.common.data.query.EntityNameFilter;
import org.thingsboard.server.common.data.query.EntitySearchQueryFilter;
@ -265,7 +266,8 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
String entityWhereClause = DefaultEntityQueryRepository.this.buildEntityWhere(ctx, query.getEntityFilter(), entityFieldsFiltersMapping);
String latestJoins = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings);
String latestJoinsCnt = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings, true);
String latestJoinsData = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings, false);
String whereClause = DefaultEntityQueryRepository.this.buildWhere(ctx, latestFiltersMapping, query.getEntityFilter().getType());
String textSearchQuery = DefaultEntityQueryRepository.this.buildTextSearchQuery(ctx, selectionMapping, pageLink.getTextSearch());
String entityFieldsSelection = EntityKeyMapping.buildSelections(entityFieldsSelectionMapping, query.getEntityFilter().getType(), entityType);
@ -286,29 +288,44 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
topSelection = topSelection + ", " + latestSelection;
}
String fromClause = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result %s",
String fromClauseCount = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result %s",
"entities.*",
entityFieldsSelection,
addEntityTableQuery(ctx, query.getEntityFilter()),
entityWhereClause,
latestJoinsCnt,
whereClause,
textSearchQuery);
String fromClauseData = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result %s",
topSelection,
entityFieldsSelection,
addEntityTableQuery(ctx, query.getEntityFilter()),
entityWhereClause,
latestJoins,
latestJoinsData,
whereClause,
textSearchQuery);
int totalElements = jdbcTemplate.queryForObject(String.format("select count(*) %s", fromClause), ctx, Integer.class);
if (!StringUtils.isEmpty(pageLink.getTextSearch())) {
//Unfortunately, we need to sacrifice performance in case of full text search, because it is applied to all joined records.
fromClauseCount = fromClauseData;
}
String countQuery = String.format("select count(id) %s", fromClauseCount);
int totalElements = jdbcTemplate.queryForObject(countQuery, ctx, Integer.class);
String dataQuery = String.format("select * %s", fromClause);
String dataQuery = String.format("select * %s", fromClauseData);
EntityDataSortOrder sortOrder = pageLink.getSortOrder();
if (sortOrder != null) {
Optional<EntityKeyMapping> sortOrderMappingOpt = mappings.stream().filter(EntityKeyMapping::isSortOrder).findFirst();
if (sortOrderMappingOpt.isPresent()) {
EntityKeyMapping sortOrderMapping = sortOrderMappingOpt.get();
dataQuery = String.format("%s order by %s", dataQuery, sortOrderMapping.getValueAlias());
if (sortOrder.getDirection() == EntityDataSortOrder.Direction.ASC) {
dataQuery += " asc";
String direction = sortOrder.getDirection() == EntityDataSortOrder.Direction.ASC ? "asc" : "desc";
if (sortOrderMapping.getEntityKey().getType() == EntityKeyType.ENTITY_FIELD) {
dataQuery = String.format("%s order by %s %s", dataQuery, sortOrderMapping.getValueAlias(), direction);
} else {
dataQuery += " desc";
dataQuery = String.format("%s order by %s %s, %s %s", dataQuery,
sortOrderMapping.getSortOrderNumAlias(), direction, sortOrderMapping.getSortOrderStrAlias(), direction);
}
}
}

51
dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java
View File

@ -270,9 +270,10 @@ public class EntityKeyMapping {
Collectors.joining(", "));
}
public static String buildLatestJoins(QueryContext ctx, EntityFilter entityFilter, EntityType entityType, List<EntityKeyMapping> latestMappings) {
return latestMappings.stream().map(mapping -> mapping.toLatestJoin(ctx, entityFilter, entityType)).collect(
Collectors.joining(" "));
public static String buildLatestJoins(QueryContext ctx, EntityFilter entityFilter, EntityType entityType, List<EntityKeyMapping> latestMappings, boolean countQuery) {
return latestMappings.stream().filter(mapping -> !countQuery || mapping.hasFilter())
.map(mapping -> mapping.toLatestJoin(ctx, entityFilter, entityType)).collect(
Collectors.joining(" "));
}
public static String buildQuery(QueryContext ctx, List<EntityKeyMapping> mappings, EntityFilterType filterType) {
@ -363,19 +364,14 @@ public class EntityKeyMapping {
}
private String buildAttributeSelection() {
String attrValAlias = getValueAlias();
String attrTsAlias = getTsAlias();
String attrValSelection =
String.format("(coalesce(cast(%s.bool_v as varchar), '') || " +
"coalesce(%s.str_v, '') || " +
"coalesce(cast(%s.long_v as varchar), '') || " +
"coalesce(cast(%s.dbl_v as varchar), '') || " +
"coalesce(cast(%s.json_v as varchar), '')) as %s", alias, alias, alias, alias, alias, attrValAlias);
String attrTsSelection = String.format("%s.last_update_ts as %s", alias, attrTsAlias);
return String.join(", ", attrValSelection, attrTsSelection);
return buildTimeSeriesOrAttrSelection(true);
}
private String buildTimeSeriesSelection() {
return buildTimeSeriesOrAttrSelection(false);
}
private String buildTimeSeriesOrAttrSelection(boolean attr) {
String attrValAlias = getValueAlias();
String attrTsAlias = getTsAlias();
String attrValSelection =
@ -384,8 +380,25 @@ public class EntityKeyMapping {
"coalesce(cast(%s.long_v as varchar), '') || " +
"coalesce(cast(%s.dbl_v as varchar), '') || " +
"coalesce(cast(%s.json_v as varchar), '')) as %s", alias, alias, alias, alias, alias, attrValAlias);
String attrTsSelection = String.format("%s.ts as %s", alias, attrTsAlias);
return String.join(", ", attrValSelection, attrTsSelection);
String attrTsSelection = String.format("%s.%s as %s", alias, attr ? "last_update_ts" : "ts", attrTsAlias);
if (this.isSortOrder) {
String attrNumAlias = getSortOrderNumAlias();
String attrVarcharAlias = getSortOrderStrAlias();
String attrSortOrderSelection =
String.format("coalesce(%s.dbl_v, cast(%s.long_v as double precision), (case when %s.bool_v then 1 else 0 end)) %s," +
"coalesce(%s.str_v, cast(%s.json_v as varchar), '') %s", alias, alias, alias, attrNumAlias, alias, alias, attrVarcharAlias);
return String.join(", ", attrValSelection, attrTsSelection, attrSortOrderSelection);
} else {
return String.join(", ", attrValSelection, attrTsSelection);
}
}
public String getSortOrderStrAlias() {
return getValueAlias() + "_so_varchar";
}
public String getSortOrderNumAlias() {
return getValueAlias() + "_so_num";
}
private String buildKeyQuery(QueryContext ctx, String alias, KeyFilter keyFilter,
@ -425,11 +438,11 @@ public class EntityKeyMapping {
if (predicate.getType().equals(FilterPredicateType.NUMERIC)) {
return this.buildNumericPredicateQuery(ctx, field, (NumericFilterPredicate) predicate);
} else if (predicate.getType().equals(FilterPredicateType.STRING)) {
if (key.getKey().equals("entityType") && !filterType.equals(EntityFilterType.RELATIONS_QUERY)){
if (key.getKey().equals("entityType") && !filterType.equals(EntityFilterType.RELATIONS_QUERY)) {
field = ctx.getEntityType().toString();
return this.buildStringPredicateQuery(ctx, field, (StringFilterPredicate) predicate)
.replace("lower(" + field, "lower('" + field + "'")
.replace(field + " ","'" + field + "' ");
.replace(field + " ", "'" + field + "' ");
} else {
return this.buildStringPredicateQuery(ctx, field, (StringFilterPredicate) predicate);
}
@ -481,13 +494,13 @@ public class EntityKeyMapping {
stringOperationQuery = String.format("%s like :%s) or (%s is null and :%s = '%%')", operationField, paramName, operationField, paramName);
break;
case CONTAINS:
if (value.length()>0) {
if (value.length() > 0) {
value = "%" + value + "%";
}
stringOperationQuery = String.format("%s like :%s) or (%s is null and :%s = '')", operationField, paramName, operationField, paramName);
break;
case NOT_CONTAINS:
if (value.length()>0) {
if (value.length() > 0) {
value = "%" + value + "%";
}
stringOperationQuery = String.format("%s not like :%s) or (%s is null and :%s != '')", operationField, paramName, operationField, paramName);

5
dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java
View File

@ -41,8 +41,9 @@ public class QueryContext implements SqlParameterSource {
}
void addParameter(String name, Object value, int type, String typeName) {
Parameter existing = params.put(name, new Parameter(value, type, typeName));
if (existing != null) {
Parameter newParam = new Parameter(value, type, typeName);
Parameter oldParam = params.put(name, newParam);
if (oldParam != null && !oldParam.value.equals(newParam.value)) {
throw new RuntimeException("Parameter with name: " + name + " was already registered!");
}
}

4
dao/src/main/resources/sql/schema-entities-idx.sql
View File

@ -36,4 +36,6 @@ CREATE INDEX IF NOT EXISTS idx_asset_customer_id ON asset(tenant_id, customer_id
CREATE INDEX IF NOT EXISTS idx_asset_customer_id_and_type ON asset(tenant_id, customer_id, type);
CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type);
CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type);
CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc);
Write Preview
Loading…
Cancel
Save