From 6b5966a57983fdcaf7963df47a32069273b81c33 Mon Sep 17 00:00:00 2001 From: desoliture Date: Mon, 7 Mar 2022 15:29:20 +0200 Subject: [PATCH 1/2] fix unresolved dynamic values in queryService on api request --- .../query/DefaultEntityQueryService.java | 83 +++++++++++++++++ common/util/pom.xml | 4 + .../org/thingsboard/common/util/KvUtil.java | 89 +++++++++++++++++++ 3 files changed, 176 insertions(+) create mode 100644 common/util/src/main/java/org/thingsboard/common/util/KvUtil.java diff --git a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java index b4337fb9f3..d2a6369c51 100644 --- a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java +++ b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java @@ -29,12 +29,16 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.common.util.KvUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; +import org.thingsboard.server.common.data.query.ComplexFilterPredicate; +import org.thingsboard.server.common.data.query.DynamicValue; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; @@ -42,6 +46,10 @@ import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.FilterPredicateType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.KeyFilterPredicate; +import org.thingsboard.server.common.data.query.SimpleKeyFilterPredicate; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; @@ -52,6 +60,7 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.subscription.TbAttributeSubscriptionScope; import java.util.ArrayList; import java.util.Collection; @@ -59,7 +68,9 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -93,9 +104,81 @@ public class DefaultEntityQueryService implements EntityQueryService { @Override public PageData findEntityDataByQuery(SecurityUser securityUser, EntityDataQuery query) { + if (query.getKeyFilters() != null) { + resolveDynamicValuesInPredicates( + query.getKeyFilters().stream() + .map(KeyFilter::getPredicate) + .collect(Collectors.toList()), + securityUser + ); + } return entityService.findEntityDataByQuery(securityUser.getTenantId(), securityUser.getCustomerId(), query); } + private void resolveDynamicValuesInPredicates(List predicates, SecurityUser user) { + predicates.forEach(predicate -> { + if (predicate.getType() == FilterPredicateType.COMPLEX) { + resolveDynamicValuesInPredicates( + ((ComplexFilterPredicate) predicate).getPredicates(), + user + ); + } else { + setResolvedValue(user, (SimpleKeyFilterPredicate) predicate); + } + }); + } + + private void setResolvedValue(SecurityUser user, SimpleKeyFilterPredicate predicate) { + DynamicValue dynamicValue = predicate.getValue().getDynamicValue(); + if (dynamicValue != null && dynamicValue.getResolvedValue() == null) { + resolveDynamicValue(dynamicValue, user, predicate.getType()); + } + } + + private void resolveDynamicValue(DynamicValue dynamicValue, SecurityUser user, FilterPredicateType predicateType) { + EntityId entityId; + switch (dynamicValue.getSourceType()) { + case CURRENT_TENANT: + entityId = user.getTenantId(); + break; + case CURRENT_CUSTOMER: + entityId = user.getCustomerId(); + break; + case CURRENT_USER: + entityId = user.getId(); + break; + default: + throw new RuntimeException("Not supported operation for source type: {" + dynamicValue.getSourceType() + "}"); + } + + try { + Optional valueOpt = attributesService.find(user.getTenantId(), entityId, + TbAttributeSubscriptionScope.SERVER_SCOPE.name(), dynamicValue.getSourceAttribute()).get(); + + if (valueOpt.isPresent()) { + AttributeKvEntry entry = valueOpt.get(); + Object resolved = null; + switch (predicateType) { + case STRING: + resolved = KvUtil.getStringValue(entry); + break; + case NUMERIC: + resolved = KvUtil.getDoubleValue(entry); + break; + case BOOLEAN: + resolved = KvUtil.getBoolValue(entry); + break; + case COMPLEX: + break; + } + + dynamicValue.setResolvedValue((T) resolved); + } + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + @Override public PageData findAlarmDataByQuery(SecurityUser securityUser, AlarmDataQuery query) { EntityDataQuery entityDataQuery = this.buildEntityDataQuery(query); diff --git a/common/util/pom.xml b/common/util/pom.xml index f7c3602bca..1bb710d746 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -84,6 +84,10 @@ awaitility test + + org.thingsboard.common + data + diff --git a/common/util/src/main/java/org/thingsboard/common/util/KvUtil.java b/common/util/src/main/java/org/thingsboard/common/util/KvUtil.java new file mode 100644 index 0000000000..ba86fda769 --- /dev/null +++ b/common/util/src/main/java/org/thingsboard/common/util/KvUtil.java @@ -0,0 +1,89 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.common.util; + +import org.thingsboard.server.common.data.kv.KvEntry; + +public class KvUtil { + + public static String getStringValue(KvEntry entry) { + switch (entry.getDataType()) { + case LONG: + return entry.getLongValue().map(String::valueOf).orElse(null); + case DOUBLE: + return entry.getDoubleValue().map(String::valueOf).orElse(null); + case BOOLEAN: + return entry.getBooleanValue().map(String::valueOf).orElse(null); + case STRING: + return entry.getStrValue().orElse(""); + case JSON: + return entry.getJsonValue().orElse(""); + default: + return null; + } + } + + public static Double getDoubleValue(KvEntry entry) { + switch (entry.getDataType()) { + case LONG: + return entry.getLongValue().map(Long::doubleValue).orElse(null); + case DOUBLE: + return entry.getDoubleValue().orElse(null); + case BOOLEAN: + return entry.getBooleanValue().map(e -> e ? 1.0 : 0).orElse(null); + case STRING: + try { + return Double.parseDouble(entry.getStrValue().orElse("")); + } catch (RuntimeException e) { + return null; + } + case JSON: + try { + return Double.parseDouble(entry.getJsonValue().orElse("")); + } catch (RuntimeException e) { + return null; + } + default: + return null; + } + } + + public static Boolean getBoolValue(KvEntry entry) { + switch (entry.getDataType()) { + case LONG: + return entry.getLongValue().map(e -> e != 0).orElse(null); + case DOUBLE: + return entry.getDoubleValue().map(e -> e != 0).orElse(null); + case BOOLEAN: + return entry.getBooleanValue().orElse(null); + case STRING: + try { + return Boolean.parseBoolean(entry.getStrValue().orElse("")); + } catch (RuntimeException e) { + return null; + } + case JSON: + try { + return Boolean.parseBoolean(entry.getJsonValue().orElse("")); + } catch (RuntimeException e) { + return null; + } + default: + return null; + } + } + +} From 89c838e560cb23aab32bb2f414ddc12b65e204f4 Mon Sep 17 00:00:00 2001 From: desoliture Date: Mon, 7 Mar 2022 15:39:18 +0200 Subject: [PATCH 2/2] add corresponding tests --- .../BaseEntityQueryControllerTest.java | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityQueryControllerTest.java index 3b5ec02441..8c692ef8a2 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityQueryControllerTest.java @@ -16,10 +16,13 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import org.awaitility.Awaitility; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; @@ -29,6 +32,8 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.DeviceTypeFilter; +import org.thingsboard.server.common.data.query.DynamicValue; +import org.thingsboard.server.common.data.query.DynamicValueSourceType; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; @@ -41,11 +46,13 @@ import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.NumericFilterPredicate; +import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.common.data.security.Authority; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -299,6 +306,79 @@ public abstract class BaseEntityQueryControllerTest extends AbstractControllerTe List deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); + } + + @Test + public void testFindEntityDataByQueryWithDynamicValue() throws Exception { + int numOfDevices = 2; + + for (int i = 0; i < numOfDevices; i++) { + Device device = new Device(); + String name = "Device" + i; + device.setName(name); + device.setType("default"); + device.setLabel("testLabel" + (int) (Math.random() * 1000)); + + Device savedDevice1 = doPost("/api/device?accessToken=" + name, device, Device.class); + JsonNode content = JacksonUtil.toJsonNode("{\"alarmActiveTime\": 1" + i + "}"); + doPost("/api/plugins/telemetry/" + EntityType.DEVICE.name() + "/" + savedDevice1.getUuidId() + "/SERVER_SCOPE", content) + .andExpect(status().isOk()); + } + JsonNode content = JacksonUtil.toJsonNode("{\"dynamicValue\": 0}"); + doPost("/api/plugins/telemetry/" + EntityType.TENANT.name() + "/" + tenantId.getId() + "/SERVER_SCOPE", content) + .andExpect(status().isOk()); + + + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceType("default"); + filter.setDeviceNameFilter(""); + + KeyFilter highTemperatureFilter = new KeyFilter(); + highTemperatureFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, "alarmActiveTime")); + NumericFilterPredicate predicate = new NumericFilterPredicate(); + + DynamicValue dynamicValue = + new DynamicValue<>(DynamicValueSourceType.CURRENT_TENANT, "dynamicValue"); + FilterPredicateValue predicateValue = new FilterPredicateValue<>(0.0, null, dynamicValue); + predicate.setValue(predicateValue); + predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + highTemperatureFilter.setPredicate(predicate); + + List keyFilters = Collections.singletonList(highTemperatureFilter); + + + EntityDataSortOrder sortOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC + ); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder); + + List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "alarmActiveTime")); + + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + + Awaitility.await() + .alias("data by query") + .atMost(30, TimeUnit.SECONDS) + .until(() -> { + var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + var loadedEntities = new ArrayList<>(data.getData()); + return loadedEntities.size() == numOfDevices; + }); + + var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + var loadedEntities = new ArrayList<>(data.getData()); + + Assert.assertEquals(numOfDevices, loadedEntities.size()); + + for (int i = 0; i < numOfDevices; i++) { + var entity = loadedEntities.get(i); + String name = entity.getLatest().get(EntityKeyType.ENTITY_FIELD).getOrDefault("name", new TsValue(0, "Invalid")).getValue(); + String alarmActiveTime = entity.getLatest().get(EntityKeyType.ATTRIBUTE).getOrDefault("alarmActiveTime", new TsValue(0, "-1")).getValue(); + + Assert.assertEquals("Device" + i, name); + Assert.assertEquals("1" + i, alarmActiveTime); + } } }