tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Merge pull request #10499 from AndriiLandiak/improve-entity-data-query 

Entity Data Query: improvement
pull/10522/head
Andrew Shvayka 2 years ago
committed by GitHub
parent
7eb825dc3b 9e1c195e59
commit
b164eea623
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
3 changed files with 213 additions and 39 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeGrpcClient.java
  2. 121
      dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java
  3. 129
      dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java

2
common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeGrpcClient.java
View File

@ -111,7 +111,7 @@ public class EdgeGrpcClient implements EdgeRpcClient {
.setConnectRequestMsg(ConnectRequestMsg.newBuilder()
.setEdgeRoutingKey(edgeKey)
.setEdgeSecret(edgeSecret)
.setEdgeVersion(EdgeVersion.V_3_6_2)
.setEdgeVersion(EdgeVersion.V_3_7_0)
.setMaxInboundMessageSize(maxInboundMessageSize)
.build())
.build());

121
dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java
View File

@ -40,11 +40,16 @@ import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityFilterType;
import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.EntityListFilter;
import org.thingsboard.server.common.data.query.KeyFilter;
import org.thingsboard.server.common.data.query.RelationsQueryFilter;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -63,6 +68,10 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe
public static final String INCORRECT_CUSTOMER_ID = "Incorrect customerId ";
public static final CustomerId NULL_CUSTOMER_ID = new CustomerId(NULL_UUID);
private static final int MAX_ENTITY_IDS_SIZE = 1024;
private static final Set<EntityFilterType> EXCLUDED_TYPES_FROM_OPTIMIZATION = Set.of(
EntityFilterType.ENTITY_LIST, EntityFilterType.SINGLE_ENTITY, EntityFilterType.RELATIONS_QUERY);
@Autowired
private EntityQueryDao entityQueryDao;
@ -86,9 +95,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe
validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id);
validateEntityDataQuery(query);
if (EntityFilterType.RELATIONS_QUERY.equals(query.getEntityFilter().getType())
|| EntityFilterType.SINGLE_ENTITY.equals(query.getEntityFilter().getType())
|| StringUtils.isNotEmpty(query.getPageLink().getTextSearch())) {
if (!isValidForOptimization(query)) {
return this.entityQueryDao.findEntityDataByQuery(tenantId, customerId, query);
}
@ -97,41 +104,10 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe
if (entityDataByQuery == null || entityDataByQuery.getData().isEmpty()) {
return entityDataByQuery;
}
// 2 step - find entity data by entity ids from the 1st step
PageData<EntityData> result = findEntityDataByEntityIds(tenantId, customerId, query, entityDataByQuery.getData());
return new PageData<>(result.getData(), entityDataByQuery.getTotalPages(), entityDataByQuery.getTotalElements(), entityDataByQuery.hasNext());
}
private PageData<EntityData> findEntityIdsByFilterAndSorterColumns(TenantId tenantId, CustomerId customerId, EntityDataQuery query) {
List<EntityKey> entityFields = null;
List<EntityKey> latestValues = null;
if (query.getPageLink().getSortOrder() != null) {
if (query.getEntityFields() != null) {
entityFields = query.getEntityFields().stream()
.filter(entityKey -> entityKey.getKey().equals(query.getPageLink().getSortOrder().getKey().getKey()))
.collect(Collectors.toList());
}
if (query.getLatestValues() != null) {
latestValues = query.getLatestValues().stream()
.filter(entityKey -> entityKey.getKey().equals(query.getPageLink().getSortOrder().getKey().getKey()))
.collect(Collectors.toList());
}
}
EntityDataQuery entityQuery = new EntityDataQuery(query.getEntityFilter(), query.getPageLink(), entityFields, latestValues, query.getKeyFilters());
return this.entityQueryDao.findEntityDataByQuery(tenantId, customerId, entityQuery);
}
private PageData<EntityData> findEntityDataByEntityIds(TenantId tenantId, CustomerId customerId, EntityDataQuery query, List<EntityData> data) {
List<String> entityIds = data.stream().map(d -> d.getEntityId().getId().toString()).toList();
EntityType entityType = data.isEmpty() ? null : data.get(0).getEntityId().getEntityType();
EntityListFilter filter = new EntityListFilter();
filter.setEntityType(entityType);
filter.setEntityList(entityIds);
EntityDataPageLink pageLink = new EntityDataPageLink(query.getPageLink().getPageSize(), 0, null, query.getPageLink().getSortOrder());
EntityDataQuery entityQuery = new EntityDataQuery(filter, pageLink, query.getEntityFields(), query.getLatestValues(), null);
return this.entityQueryDao.findEntityDataByQuery(tenantId, customerId, entityQuery);
// 2 step - find entity data by entity ids from the 1st step
List<EntityData> result = fetchEntityDataByIdsFromInitialQuery(tenantId, customerId, query, entityDataByQuery.getData());
return new PageData<>(result, entityDataByQuery.getTotalPages(), entityDataByQuery.getTotalElements(), entityDataByQuery.hasNext());
}
@Override
@ -228,4 +204,75 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe
}
}
private boolean isValidForOptimization(EntityDataQuery query) {
if (StringUtils.isNotEmpty(query.getPageLink().getTextSearch())) {
return false;
}
if (EXCLUDED_TYPES_FROM_OPTIMIZATION.contains(query.getEntityFilter().getType())) {
return false;
}
if ((query.getEntityFields() == null || query.getEntityFields().isEmpty()) &&
(query.getLatestValues() == null || query.getLatestValues().isEmpty())) {
return false;
}
Set<EntityKey> filteringKeys = new HashSet<>(Optional.ofNullable(query.getKeyFilters()).orElse(Collections.emptyList()).stream().map(KeyFilter::getKey).toList());
Set<EntityKey> entityFields = new HashSet<>(Optional.ofNullable(query.getEntityFields()).orElse(Collections.emptyList()));
Set<EntityKey> latestValues = new HashSet<>(Optional.ofNullable(query.getLatestValues()).orElse(Collections.emptyList()));
return !(filteringKeys.containsAll(entityFields) && filteringKeys.containsAll(latestValues));
}
private PageData<EntityData> findEntityIdsByFilterAndSorterColumns(TenantId tenantId, CustomerId customerId, EntityDataQuery query) {
List<EntityKey> entityFields = null;
List<EntityKey> latestValues = null;
if (query.getPageLink().getSortOrder() != null) {
if (query.getEntityFields() != null) {
entityFields = query.getEntityFields().stream()
.filter(entityKey -> entityKey.getKey().equals(query.getPageLink().getSortOrder().getKey().getKey()))
.collect(Collectors.toList());
}
if (query.getLatestValues() != null) {
latestValues = query.getLatestValues().stream()
.filter(entityKey -> entityKey.getKey().equals(query.getPageLink().getSortOrder().getKey().getKey()))
.collect(Collectors.toList());
}
}
EntityDataQuery entityQuery = new EntityDataQuery(query.getEntityFilter(), query.getPageLink(), entityFields, latestValues, query.getKeyFilters());
return this.entityQueryDao.findEntityDataByQuery(tenantId, customerId, entityQuery);
}
private List<EntityData> fetchEntityDataByIdsFromInitialQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query, List<EntityData> initialQueryResult) {
List<EntityData> result = new ArrayList<>();
List<String> entityIds = initialQueryResult.stream().map(d -> d.getEntityId().getId().toString()).collect(Collectors.toList());
EntityType entityType = initialQueryResult.get(0).getEntityId().getEntityType();
if (entityIds.size() > MAX_ENTITY_IDS_SIZE) {
List<List<String>> chunks = new ArrayList<>();
for (int i = 0; i < entityIds.size(); i += MAX_ENTITY_IDS_SIZE) {
chunks.add(entityIds.subList(i, Math.min(entityIds.size(), i + MAX_ENTITY_IDS_SIZE)));
}
for (List<String> chunk : chunks) {
result.addAll(findEntityDataByEntityIds(tenantId, customerId, query, chunk, entityType, chunk.size()));
}
} else {
result.addAll(findEntityDataByEntityIds(tenantId, customerId, query, entityIds, entityType, query.getPageLink().getPageSize()));
}
return result;
}
private List<EntityData> findEntityDataByEntityIds(TenantId tenantId, CustomerId customerId, EntityDataQuery query,
List<String> entityIds, EntityType entityType, int pageSize) {
EntityListFilter filter = new EntityListFilter();
filter.setEntityType(entityType);
filter.setEntityList(entityIds);
EntityDataPageLink pageLink = new EntityDataPageLink(pageSize, 0, null, query.getPageLink().getSortOrder());
EntityDataQuery entityQuery = new EntityDataQuery(filter, pageLink, query.getEntityFields(), query.getLatestValues(), null);
return this.entityQueryDao.findEntityDataByQuery(tenantId, customerId, entityQuery).getData();
}
}

129
dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java
View File

@ -26,7 +26,6 @@ import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.ResultSetExtractor;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.StringUtils;
@ -99,6 +98,8 @@ import java.util.stream.Stream;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
import static org.thingsboard.server.common.data.query.EntityKeyType.ATTRIBUTE;
import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD;
@Slf4j
@DaoSqlTest
@ -2113,6 +2114,132 @@ public class EntityServiceTest extends AbstractServiceTest {
deviceService.deleteDevicesByTenantId(tenantId);
}
@Test
public void testFindEntityQuery_for_5000_devices_with_3000_pageSize() {
int pageSize = 3000;
int expectedDevicesSize = 4000;
int unexpectedDevicesSize = 1000;
for (int i = 0; i < expectedDevicesSize + unexpectedDevicesSize; i++) {
Device device = new Device();
device.setTenantId(tenantId);
if (i < expectedDevicesSize) {
device.setName("Device_" + i); // match deviceNameFilter 'D%'
} else {
device.setName("Test_" + i); // does not match deviceNameFilter 'D%'
}
device.setType("default");
device.setLabel("testLabel" + (int) (Math.random() * 1000));
Device savedDevice = deviceService.saveDevice(device);
attributesService.save(tenantId, savedDevice.getId(), AttributeScope.CLIENT_SCOPE,
new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("telemetry", (long) i)));
}
DeviceTypeFilter filter = new DeviceTypeFilter();
filter.setDeviceTypes(List.of("default"));
filter.setDeviceNameFilter("D%");
EntityDataSortOrder sortOrder = new EntityDataSortOrder(new EntityKey(ATTRIBUTE, "telemetry"), EntityDataSortOrder.Direction.DESC);
List<KeyFilter> deviceTypeFilters = createStringKeyFilters("type", ENTITY_FIELD, StringFilterPredicate.StringOperation.EQUAL, "default");
List<KeyFilter> attributeFilters = Collections.singletonList(createNumericKeyFilter("telemetry", ATTRIBUTE, NumericFilterPredicate.NumericOperation.LESS, expectedDevicesSize));
List<KeyFilter> nameFilters = createStringKeyFilters("name", ENTITY_FIELD, StringFilterPredicate.StringOperation.CONTAINS, "Device");
List<EntityKey> entityFields = Arrays.asList(new EntityKey(ENTITY_FIELD, "name"), new EntityKey(ENTITY_FIELD, "type"));
// 1. Device type filters:
// query with textSearch - optimization is not performing
EntityDataPageLink originalPageLink = new EntityDataPageLink(pageSize, 0, "Device", sortOrder);
EntityDataQuery originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, deviceTypeFilters);
PageData<EntityData> originalData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), originalQuery);
// query without textSearch - optimization is performing
EntityDataPageLink optimizedPageLink = new EntityDataPageLink(pageSize, 0, null, sortOrder);
EntityDataQuery optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, deviceTypeFilters);
PageData<EntityData> optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery);
List<EntityData> loadedEntities = getLoadedEntities(optimizedData, optimizedQuery);
Assert.assertEquals(expectedDevicesSize, loadedEntities.size());
loadedEntities = getLoadedEntities(originalData, originalQuery);
Assert.assertEquals(expectedDevicesSize, loadedEntities.size());
Assert.assertEquals(pageSize, optimizedData.getData().size());
for (int i = 0; i < pageSize; i++) {
EntityData originalElement = originalData.getData().get(i);
EntityData optimizedElement = optimizedData.getData().get(i);
Assert.assertEquals(originalElement.getEntityId(), optimizedElement.getEntityId());
originalElement.getLatest().get(ENTITY_FIELD).forEach((key, value) -> {
Assert.assertEquals(value.getValue(), optimizedElement.getLatest().get(EntityKeyType.ENTITY_FIELD).get(key).getValue());
Assert.assertEquals(value.getCount(), optimizedElement.getLatest().get(EntityKeyType.ENTITY_FIELD).get(key).getCount());
});
}
Assert.assertEquals(originalData.getTotalPages(), optimizedData.getTotalPages());
Assert.assertEquals(originalData.getTotalElements(), optimizedData.getTotalElements());
// 2. Device attribute filters
// query with textSearch - optimization is not performing
originalPageLink = new EntityDataPageLink(pageSize, 0, "Device", sortOrder);
originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, attributeFilters);
originalData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), originalQuery);
// query without textSearch - optimization is performing
optimizedPageLink = new EntityDataPageLink(pageSize, 0, null, sortOrder);
optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, attributeFilters);
optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery);
loadedEntities = getLoadedEntities(optimizedData, optimizedQuery);
Assert.assertEquals(expectedDevicesSize, loadedEntities.size());
loadedEntities = getLoadedEntities(originalData, originalQuery);
Assert.assertEquals(expectedDevicesSize, loadedEntities.size());
Assert.assertEquals(pageSize, optimizedData.getData().size());
for (int i = 0; i < pageSize; i++) {
EntityData originalElement = originalData.getData().get(i);
EntityData optimizedElement = optimizedData.getData().get(i);
Assert.assertEquals(originalElement.getEntityId(), optimizedElement.getEntityId());
originalElement.getLatest().get(ENTITY_FIELD).forEach((key, value) -> {
Assert.assertEquals(value.getValue(), optimizedElement.getLatest().get(EntityKeyType.ENTITY_FIELD).get(key).getValue());
Assert.assertEquals(value.getCount(), optimizedElement.getLatest().get(EntityKeyType.ENTITY_FIELD).get(key).getCount());
});
}
Assert.assertEquals(originalData.getTotalPages(), optimizedData.getTotalPages());
Assert.assertEquals(originalData.getTotalElements(), optimizedData.getTotalElements());
// 3. Device name filters
// query with textSearch - optimization is not performing
originalPageLink = new EntityDataPageLink(pageSize, 0, "Device", sortOrder);
originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, nameFilters);
originalData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), originalQuery);
// query without textSearch - optimization is performing
optimizedPageLink = new EntityDataPageLink(pageSize, 0, null, sortOrder);
optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, nameFilters);
optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery);
loadedEntities = getLoadedEntities(optimizedData, optimizedQuery);
Assert.assertEquals(expectedDevicesSize, loadedEntities.size());
loadedEntities = getLoadedEntities(originalData, originalQuery);
Assert.assertEquals(expectedDevicesSize, loadedEntities.size());
Assert.assertEquals(pageSize, optimizedData.getData().size());
for (int i = 0; i < pageSize; i++) {
EntityData originalElement = originalData.getData().get(i);
EntityData optimizedElement = optimizedData.getData().get(i);
Assert.assertEquals(originalElement.getEntityId(), optimizedElement.getEntityId());
originalElement.getLatest().get(ENTITY_FIELD).forEach((key, value) -> {
Assert.assertEquals(value.getValue(), optimizedElement.getLatest().get(EntityKeyType.ENTITY_FIELD).get(key).getValue());
Assert.assertEquals(value.getCount(), optimizedElement.getLatest().get(EntityKeyType.ENTITY_FIELD).get(key).getCount());
});
}
Assert.assertEquals(originalData.getTotalPages(), optimizedData.getTotalPages());
Assert.assertEquals(originalData.getTotalElements(), optimizedData.getTotalElements());
deviceService.deleteDevicesByTenantId(tenantId);
}
private Boolean listEqualWithoutOrder(List<String> A, List<String> B) {
return A.containsAll(B) && B.containsAll(A);
}

Write Preview
Loading…
Cancel
Save