tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Improve data query

pull/2989/head
Igor Kulikov 6 years ago
parent
42bf0a37a2
commit
eebcccc6fd
4 changed files with 197 additions and 72 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 30
      dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java
  2. 32
      dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityDataAdapter.java
  3. 97
      dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java
  4. 110
      dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java

30
dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java
View File

@ -114,25 +114,23 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
entityFieldsSelection = String.format("e.id, '%s'", entityType.name());
}
String latestSelection = EntityKeyMapping.buildSelections(latestSelectionMapping);
String selection = entityFieldsSelection;
String topSelection = "entities.*";
if (!StringUtils.isEmpty(latestSelection)) {
selection = entityFieldsSelection + ", " + latestSelection;
topSelection = topSelection + ", " + latestSelection;
}
String fromClause = String.format("from (select %s from %s e where %s) entities %s",
selection,
String fromClause = String.format("from (select %s from (select %s from %s e where %s) entities %s %s) result",
topSelection,
entityFieldsSelection,
entityTableMap.get(entityType),
entityWhereClause,
latestJoins);
if (!StringUtils.isEmpty(whereClause)) {
fromClause = String.format("%s where %s", fromClause, whereClause);
}
latestJoins,
whereClause);
int totalElements = ((BigInteger)entityManager.createNativeQuery(String.format("select count(*) %s", fromClause))
.getSingleResult()).intValue();
String dataQuery = String.format("select entities.* %s", fromClause);
String dataQuery = String.format("select * %s", fromClause);
EntityDataSortOrder sortOrder = pageLink.getSortOrder();
if (sortOrder != null) {
@ -198,12 +196,18 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
private String buildWhere(List<EntityKeyMapping> selectionMapping, List<EntityKeyMapping> latestFiltersMapping, String searchText) {
String latestFilters = EntityKeyMapping.buildQuery(latestFiltersMapping);
String textSearchQuery = this.buildTextSearchQuery(selectionMapping, searchText);
String query;
if (!StringUtils.isEmpty(latestFilters) && !StringUtils.isEmpty(textSearchQuery)) {
return String.join(" AND ", latestFilters, textSearchQuery);
query = String.join(" AND ", latestFilters, textSearchQuery);
} else if (!StringUtils.isEmpty(latestFilters)) {
return latestFilters;
query = latestFilters;
} else {
query = textSearchQuery;
}
if (!StringUtils.isEmpty(query)) {
return String.format("where %s", query);
} else {
return textSearchQuery;
return "";
}
}

32
dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityDataAdapter.java
View File

@ -56,20 +56,22 @@ public class EntityDataAdapter {
Map<String, TsValue[]> timeseries = new HashMap<>();
EntityData entityData = new EntityData(entityId, latest, timeseries);
for (EntityKeyMapping mapping: selectionMapping) {
EntityKey entityKey = mapping.getEntityKey();
Object value = row[mapping.getIndex()];
String strValue;
long ts;
if (entityKey.getType().equals(EntityKeyType.ENTITY_FIELD)) {
strValue = value != null ? value.toString() : null;
ts = System.currentTimeMillis();
} else {
strValue = convertValue(value);
Object tsObject = row[mapping.getIndex()+1];
ts = Long.parseLong(tsObject.toString());
if (!mapping.isIgnore()) {
EntityKey entityKey = mapping.getEntityKey();
Object value = row[mapping.getIndex()];
String strValue;
long ts;
if (entityKey.getType().equals(EntityKeyType.ENTITY_FIELD)) {
strValue = value != null ? value.toString() : null;
ts = System.currentTimeMillis();
} else {
strValue = convertValue(value);
Object tsObject = row[mapping.getIndex() + 1];
ts = Long.parseLong(tsObject.toString());
}
TsValue tsValue = new TsValue(ts, strValue);
latest.computeIfAbsent(entityKey.getType(), entityKeyType -> new HashMap<>()).put(entityKey.getKey(), tsValue);
}
TsValue tsValue = new TsValue(ts, strValue);
latest.computeIfAbsent(entityKey.getType(), entityKeyType -> new HashMap<>()).put(entityKey.getKey(), tsValue);
}
return entityData;
}
@ -80,8 +82,8 @@ public class EntityDataAdapter {
// check number
if (strVal.length() > 0) {
try {
int intVal = Integer.parseInt(strVal);
return Integer.toString(intVal);
long longVal = Long.parseLong(strVal);
return Long.toString(longVal);
} catch (NumberFormatException ignored) {
}
try {

97
dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java
View File

@ -35,6 +35,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -65,6 +66,7 @@ public class EntityKeyMapping {
private boolean isLatest;
private boolean isSelection;
private boolean isSortOrder;
private boolean ignore = false;
private List<KeyFilter> keyFilters;
private EntityKey entityKey;
@ -151,42 +153,62 @@ public class EntityKeyMapping {
query.getKeyFilters().stream().collect(Collectors.groupingBy(KeyFilter::getKey)) : Collections.emptyMap();
EntityDataSortOrder sortOrder = query.getPageLink().getSortOrder();
EntityKey sortOrderKey = sortOrder != null ? sortOrder.getKey() : null;
EntityKeyMapping sortOrderMapping = null;
int index = 2;
List<EntityKeyMapping> mappings = new ArrayList<>();
for (EntityKey entityField : entityFields) {
EntityKeyMapping mapping = new EntityKeyMapping();
mapping.setIndex(index);
mapping.setAlias(String.format("alias%s", index));
mapping.setKeyFilters(filters.remove(entityField));
mapping.setLatest(false);
mapping.setSelection(true);
mapping.setEntityKey(entityField);
if (entityField.equals(sortOrderKey)) {
mapping.setSortOrder(true);
sortOrderMapping = mapping;
List<EntityKeyMapping> entityFieldsMappings = entityFields.stream().map(
key -> {
EntityKeyMapping mapping = new EntityKeyMapping();
mapping.setLatest(false);
mapping.setSelection(true);
mapping.setEntityKey(key);
return mapping;
}
).collect(Collectors.toList());
List<EntityKeyMapping> latestMappings = latestValues.stream().map(
key -> {
EntityKeyMapping mapping = new EntityKeyMapping();
mapping.setLatest(true);
mapping.setSelection(true);
mapping.setEntityKey(key);
return mapping;
}
).collect(Collectors.toList());
if (sortOrderKey != null) {
Optional<EntityKeyMapping> existing;
if (sortOrderKey.getType().equals(EntityKeyType.ENTITY_FIELD)) {
existing =
entityFieldsMappings.stream().filter(mapping -> mapping.entityKey.equals(sortOrderKey)).findFirst();
} else {
existing =
latestMappings.stream().filter(mapping -> mapping.entityKey.equals(sortOrderKey)).findFirst();
}
if (existing.isPresent()) {
existing.get().setSortOrder(true);
} else {
mapping.setSortOrder(false);
EntityKeyMapping sortOrderMapping = new EntityKeyMapping();
sortOrderMapping.setLatest(!sortOrderKey.getType().equals(EntityKeyType.ENTITY_FIELD));
sortOrderMapping.setSelection(true);
sortOrderMapping.setEntityKey(sortOrderKey);
sortOrderMapping.setSortOrder(true);
sortOrderMapping.setIgnore(true);
if (sortOrderKey.getType().equals(EntityKeyType.ENTITY_FIELD)) {
entityFieldsMappings.add(sortOrderMapping);
} else {
latestMappings.add(sortOrderMapping);
}
}
mappings.add(mapping);
index++;
}
for (EntityKey latestField : latestValues) {
EntityKeyMapping mapping = new EntityKeyMapping();
List<EntityKeyMapping> mappings = new ArrayList<>();
mappings.addAll(entityFieldsMappings);
mappings.addAll(latestMappings);
for (EntityKeyMapping mapping : mappings) {
mapping.setIndex(index);
mapping.setAlias(String.format("alias%s", index));
mapping.setKeyFilters(filters.remove(latestField));
mapping.setLatest(true);
mapping.setSelection(true);
mapping.setEntityKey(latestField);
if (latestField.equals(sortOrderKey)) {
mapping.setSortOrder(true);
sortOrderMapping = mapping;
mapping.setKeyFilters(filters.remove(mapping.entityKey));
if (mapping.getEntityKey().getType().equals(EntityKeyType.ENTITY_FIELD)) {
index++;
} else {
mapping.setSortOrder(false);
index +=2;
}
mappings.add(mapping);
index +=2;
}
if (!filters.isEmpty()) {
for (EntityKey filterField : filters.keySet()) {
@ -201,37 +223,28 @@ public class EntityKeyMapping {
index +=1;
}
}
if (sortOrderKey != null && sortOrderMapping == null) {
sortOrderMapping = new EntityKeyMapping();
sortOrderMapping.setIndex(index);
sortOrderMapping.setAlias(String.format("alias%s", index));
sortOrderMapping.setLatest(!sortOrderKey.getType().equals(EntityKeyType.ENTITY_FIELD));
sortOrderMapping.setSelection(true);
sortOrderMapping.setEntityKey(sortOrderKey);
sortOrderMapping.setSortOrder(true);
mappings.add(sortOrderMapping);
}
return mappings;
}
private String buildAttributeSelection() {
String attrValAlias = getValueAlias();
String attrTsAlias = getTsAlias();
String attrTsSelection = String.format("%s.last_update_ts as %s", alias, attrTsAlias);
String attrValSelection =
String.format("coalesce(cast(%s.bool_v as varchar), '') || " +
String.format("(coalesce(cast(%s.bool_v as varchar), '') || " +
"coalesce(%s.str_v, '') || " +
"coalesce(cast(%s.long_v as varchar), '') || " +
"coalesce(cast(%s.dbl_v as varchar), '') || " +
"coalesce(cast(%s.json_v as varchar), '')) as %s", alias, alias, alias, alias, alias, attrValAlias);
return String.join(", ", attrTsSelection, attrValSelection);
String attrTsSelection = String.format("%s.last_update_ts as %s", alias, attrTsAlias);
return String.join(", ", attrValSelection, attrTsSelection);
}
private String buildTimeseriesSelection() {
// TODO:
String attrValAlias = getValueAlias();
String attrTsAlias = getTsAlias();
return String.format("(select 1) as %s, (select '') as %s", attrTsAlias, attrValAlias);
return String.format("(select '') as %s, (select 1) as %s", attrValAlias, attrTsAlias);
}
private String buildKeyQuery(String alias, KeyFilter keyFilter) {

110
dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java
View File

@ -15,10 +15,14 @@
*/
package org.thingsboard.server.dao.service;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.Tenant;
@ -26,6 +30,10 @@ import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.DeviceTypeFilter;
import org.thingsboard.server.common.data.query.EntityCountQuery;
@ -37,14 +45,21 @@ import org.thingsboard.server.common.data.query.EntityFilter;
import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.EntityListFilter;
import org.thingsboard.server.common.data.query.KeyFilter;
import org.thingsboard.server.common.data.query.NumericFilterPredicate;
import org.thingsboard.server.dao.attributes.AttributesService;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
public abstract class BaseEntityServiceTest extends AbstractServiceTest {
@Autowired
private AttributesService attributesService;
private TenantId tenantId;
@Before
@ -105,7 +120,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
}
@Test
public void testFindEntityDataByQuery() {
public void testSimpleFindEntityDataByQuery() {
List<Device> devices = new ArrayList<>();
for (int i = 0; i < 97; i++) {
Device device = new Device();
@ -135,7 +150,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
Assert.assertEquals(10, data.getData().size());
List<EntityData> loadedEntities = new ArrayList<>(data.getData());
while(data.hasNext()) {
while (data.hasNext()) {
query = query.next();
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities.addAll(data.getData());
@ -162,5 +177,96 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
Assert.assertEquals(11, data.getTotalElements());
Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue());
deviceService.deleteDevicesByTenantId(tenantId);
}
@Test
public void testFindEntityDataByQueryWithAttributes() throws ExecutionException, InterruptedException {
List<Device> devices = new ArrayList<>();
List<Long> temperatures = new ArrayList<>();
List<Long> highTemperatures = new ArrayList<>();
for (int i=0;i<67;i++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("Device"+i);
device.setType("default");
device.setLabel("testLabel"+(int)(Math.random()*1000));
devices.add(deviceService.saveDevice(device));
long temperature = (long)(Math.random()*100);
temperatures.add(temperature);
if (temperature > 45) {
highTemperatures.add(temperature);
}
}
List<ListenableFuture<List<Void>>> attributeFutures = new ArrayList<>();
for (int i=0;i<devices.size();i++) {
Device device = devices.get(i);
attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), DataConstants.CLIENT_SCOPE));
}
Futures.successfulAsList(attributeFutures).get();
DeviceTypeFilter filter = new DeviceTypeFilter();
filter.setDeviceType("default");
filter.setDeviceNameFilter("");
EntityDataSortOrder sortOrder = new EntityDataSortOrder(
new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC
);
EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
List<EntityKey> entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"));
List<EntityKey> latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature"));
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null);
PageData<EntityData> data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
List<EntityData> loadedEntities = new ArrayList<>(data.getData());
while (data.hasNext()) {
query = query.next();
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities.addAll(data.getData());
}
Assert.assertEquals(67, loadedEntities.size());
List<String> loadedTemperatures = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList());
List<String> deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
Assert.assertEquals(deviceTemperatures, loadedTemperatures);
pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
KeyFilter highTemperatureFilter = new KeyFilter();
highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature"));
NumericFilterPredicate predicate = new NumericFilterPredicate();
predicate.setValue(45);
predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);
highTemperatureFilter.setPredicate(predicate);
List<KeyFilter> keyFilters = Collections.singletonList(highTemperatureFilter);
query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = new ArrayList<>(data.getData());
while (data.hasNext()) {
query = query.next();
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities.addAll(data.getData());
}
Assert.assertEquals(highTemperatures.size(), loadedEntities.size());
List<String> loadedHighTemperatures = loadedEntities.stream().map(entityData ->
entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList());
List<String> deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures);
deviceService.deleteDevicesByTenantId(tenantId);
}
private ListenableFuture<List<Void>> saveLongAttribute(EntityId entityId, String key, long value, String scope) {
KvEntry attrValue = new LongDataEntry(key, value);
AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L);
return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr));
}
}

Write Preview
Loading…
Cancel
Save