Merge pull request #6213 from desoliture1/unresolverDynamicValuesOnExportDevicesFilterFix

[3.3.4] Fix unresolved dynamic values on API request

application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java

@@ -29,12 +29,16 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 import org.springframework.web.context.request.async.DeferredResult;
+import org.thingsboard.common.util.KvUtil;
 import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.kv.AttributeKvEntry;
 import org.thingsboard.server.common.data.page.PageData;
 import org.thingsboard.server.common.data.query.AlarmData;
 import org.thingsboard.server.common.data.query.AlarmDataQuery;
+import org.thingsboard.server.common.data.query.ComplexFilterPredicate;
+import org.thingsboard.server.common.data.query.DynamicValue;
 import org.thingsboard.server.common.data.query.EntityCountQuery;
 import org.thingsboard.server.common.data.query.EntityData;
 import org.thingsboard.server.common.data.query.EntityDataPageLink;
@@ -42,6 +46,10 @@ import org.thingsboard.server.common.data.query.EntityDataQuery;
 import org.thingsboard.server.common.data.query.EntityDataSortOrder;
 import org.thingsboard.server.common.data.query.EntityKey;
 import org.thingsboard.server.common.data.query.EntityKeyType;
+import org.thingsboard.server.common.data.query.FilterPredicateType;
+import org.thingsboard.server.common.data.query.KeyFilter;
+import org.thingsboard.server.common.data.query.KeyFilterPredicate;
+import org.thingsboard.server.common.data.query.SimpleKeyFilterPredicate;
 import org.thingsboard.server.dao.alarm.AlarmService;
 import org.thingsboard.server.dao.attributes.AttributesService;
 import org.thingsboard.server.dao.entity.EntityService;
@@ -52,6 +60,7 @@ import org.thingsboard.server.queue.util.TbCoreComponent;
 import org.thingsboard.server.service.executors.DbCallbackExecutorService;
 import org.thingsboard.server.service.security.AccessValidator;
 import org.thingsboard.server.service.security.model.SecurityUser;
+import org.thingsboard.server.service.subscription.TbAttributeSubscriptionScope;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -59,7 +68,9 @@ import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.ExecutionException;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
@@ -93,9 +104,81 @@ public class DefaultEntityQueryService implements EntityQueryService {
 
     @Override
     public PageData<EntityData> findEntityDataByQuery(SecurityUser securityUser, EntityDataQuery query) {
+        if (query.getKeyFilters() != null) {
+            resolveDynamicValuesInPredicates(
+                    query.getKeyFilters().stream()
+                            .map(KeyFilter::getPredicate)
+                            .collect(Collectors.toList()),
+                    securityUser
+            );
+        }
         return entityService.findEntityDataByQuery(securityUser.getTenantId(), securityUser.getCustomerId(), query);
     }
 
+    private void resolveDynamicValuesInPredicates(List<KeyFilterPredicate> predicates, SecurityUser user) {
+        predicates.forEach(predicate -> {
+            if (predicate.getType() == FilterPredicateType.COMPLEX) {
+                resolveDynamicValuesInPredicates(
+                        ((ComplexFilterPredicate) predicate).getPredicates(),
+                        user
+                );
+            } else {
+                setResolvedValue(user, (SimpleKeyFilterPredicate<?>) predicate);
+            }
+        });
+    }
+
+    private void setResolvedValue(SecurityUser user, SimpleKeyFilterPredicate<?> predicate) {
+        DynamicValue<?> dynamicValue = predicate.getValue().getDynamicValue();
+        if (dynamicValue != null && dynamicValue.getResolvedValue() == null) {
+            resolveDynamicValue(dynamicValue, user, predicate.getType());
+        }
+    }
+
+    private <T> void resolveDynamicValue(DynamicValue<T> dynamicValue, SecurityUser user, FilterPredicateType predicateType) {
+        EntityId entityId;
+        switch (dynamicValue.getSourceType()) {
+            case CURRENT_TENANT:
+                entityId = user.getTenantId();
+                break;
+            case CURRENT_CUSTOMER:
+                entityId = user.getCustomerId();
+                break;
+            case CURRENT_USER:
+                entityId = user.getId();
+                break;
+            default:
+                throw new RuntimeException("Not supported operation for source type: {" + dynamicValue.getSourceType() + "}");
+        }
+
+        try {
+            Optional<AttributeKvEntry> valueOpt = attributesService.find(user.getTenantId(), entityId,
+                    TbAttributeSubscriptionScope.SERVER_SCOPE.name(), dynamicValue.getSourceAttribute()).get();
+
+            if (valueOpt.isPresent()) {
+                AttributeKvEntry entry = valueOpt.get();
+                Object resolved = null;
+                switch (predicateType) {
+                    case STRING:
+                        resolved = KvUtil.getStringValue(entry);
+                        break;
+                    case NUMERIC:
+                        resolved = KvUtil.getDoubleValue(entry);
+                        break;
+                    case BOOLEAN:
+                        resolved = KvUtil.getBoolValue(entry);
+                        break;
+                    case COMPLEX:
+                        break;
+                }
+
+                dynamicValue.setResolvedValue((T) resolved);
+            }
+        } catch (InterruptedException | ExecutionException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     @Override
     public PageData<AlarmData> findAlarmDataByQuery(SecurityUser securityUser, AlarmDataQuery query) {
         EntityDataQuery entityDataQuery = this.buildEntityDataQuery(query);

application/src/test/java/org/thingsboard/server/controller/BaseEntityQueryControllerTest.java

@@ -16,10 +16,13 @@
 package org.thingsboard.server.controller;
 
 import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import org.awaitility.Awaitility;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.server.common.data.DataConstants;
 import org.thingsboard.server.common.data.Device;
 import org.thingsboard.server.common.data.EntityType;
@@ -29,6 +32,8 @@ import org.thingsboard.server.common.data.id.DeviceId;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.page.PageData;
 import org.thingsboard.server.common.data.query.DeviceTypeFilter;
+import org.thingsboard.server.common.data.query.DynamicValue;
+import org.thingsboard.server.common.data.query.DynamicValueSourceType;
 import org.thingsboard.server.common.data.query.EntityCountQuery;
 import org.thingsboard.server.common.data.query.EntityData;
 import org.thingsboard.server.common.data.query.EntityDataPageLink;
@@ -41,11 +46,13 @@ import org.thingsboard.server.common.data.query.EntityTypeFilter;
 import org.thingsboard.server.common.data.query.FilterPredicateValue;
 import org.thingsboard.server.common.data.query.KeyFilter;
 import org.thingsboard.server.common.data.query.NumericFilterPredicate;
+import org.thingsboard.server.common.data.query.TsValue;
 import org.thingsboard.server.common.data.security.Authority;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@@ -299,6 +306,79 @@ public abstract class BaseEntityQueryControllerTest extends AbstractControllerTe
         List<String> deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
 
         Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures);
+    }
+
+    @Test
+    public void testFindEntityDataByQueryWithDynamicValue() throws Exception {
+        int numOfDevices = 2;
+
+        for (int i = 0; i < numOfDevices; i++) {
+            Device device = new Device();
+            String name = "Device" + i;
+            device.setName(name);
+            device.setType("default");
+            device.setLabel("testLabel" + (int) (Math.random() * 1000));
+
+            Device savedDevice1 = doPost("/api/device?accessToken=" + name, device, Device.class);
+            JsonNode content = JacksonUtil.toJsonNode("{\"alarmActiveTime\": 1" + i + "}");
+            doPost("/api/plugins/telemetry/" + EntityType.DEVICE.name() + "/" + savedDevice1.getUuidId() + "/SERVER_SCOPE", content)
+                    .andExpect(status().isOk());
+        }
+        JsonNode content = JacksonUtil.toJsonNode("{\"dynamicValue\": 0}");
+        doPost("/api/plugins/telemetry/" + EntityType.TENANT.name() + "/" + tenantId.getId() + "/SERVER_SCOPE", content)
+                .andExpect(status().isOk());
+
+
+        DeviceTypeFilter filter = new DeviceTypeFilter();
+        filter.setDeviceType("default");
+        filter.setDeviceNameFilter("");
+
+        KeyFilter highTemperatureFilter = new KeyFilter();
+        highTemperatureFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, "alarmActiveTime"));
+        NumericFilterPredicate predicate = new NumericFilterPredicate();
+
+        DynamicValue<Double> dynamicValue =
+                new DynamicValue<>(DynamicValueSourceType.CURRENT_TENANT, "dynamicValue");
+        FilterPredicateValue<Double> predicateValue = new FilterPredicateValue<>(0.0, null, dynamicValue);
 
+        predicate.setValue(predicateValue);
+        predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);
+        highTemperatureFilter.setPredicate(predicate);
+
+        List<KeyFilter> keyFilters = Collections.singletonList(highTemperatureFilter);
+
+
+        EntityDataSortOrder sortOrder = new EntityDataSortOrder(
+                new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC
+        );
+        EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
+
+        List<EntityKey> entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"));
+        List<EntityKey> latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "alarmActiveTime"));
+
+        EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters);
+
+        Awaitility.await()
+                .alias("data by query")
+                .atMost(30, TimeUnit.SECONDS)
+                .until(() -> {
+                    var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {});
+                    var loadedEntities = new ArrayList<>(data.getData());
+                    return loadedEntities.size() == numOfDevices;
+                });
+
+        var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {});
+        var loadedEntities = new ArrayList<>(data.getData());
+
+        Assert.assertEquals(numOfDevices, loadedEntities.size());
+
+        for (int i = 0; i < numOfDevices; i++) {
+            var entity = loadedEntities.get(i);
+            String name = entity.getLatest().get(EntityKeyType.ENTITY_FIELD).getOrDefault("name", new TsValue(0, "Invalid")).getValue();
+            String alarmActiveTime = entity.getLatest().get(EntityKeyType.ATTRIBUTE).getOrDefault("alarmActiveTime", new TsValue(0, "-1")).getValue();
+
+            Assert.assertEquals("Device" + i, name);
+            Assert.assertEquals("1" + i, alarmActiveTime);
+        }
     }
 }

common/util/pom.xml

@@ -84,6 +84,10 @@
             <artifactId>awaitility</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.thingsboard.common</groupId>
+            <artifactId>data</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

common/util/src/main/java/org/thingsboard/common/util/KvUtil.java

@@ -0,0 +1,89 @@
+/**
+ * Copyright © 2016-2022 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.common.util;
+
+import org.thingsboard.server.common.data.kv.KvEntry;
+
+public class KvUtil {
+
+    public static String getStringValue(KvEntry entry) {
+        switch (entry.getDataType()) {
+            case LONG:
+                return entry.getLongValue().map(String::valueOf).orElse(null);
+            case DOUBLE:
+                return entry.getDoubleValue().map(String::valueOf).orElse(null);
+            case BOOLEAN:
+                return entry.getBooleanValue().map(String::valueOf).orElse(null);
+            case STRING:
+                return entry.getStrValue().orElse("");
+            case JSON:
+                return entry.getJsonValue().orElse("");
+            default:
+                return null;
+        }
+    }
+
+    public static Double getDoubleValue(KvEntry entry) {
+        switch (entry.getDataType()) {
+            case LONG:
+                return entry.getLongValue().map(Long::doubleValue).orElse(null);
+            case DOUBLE:
+                return entry.getDoubleValue().orElse(null);
+            case BOOLEAN:
+                return entry.getBooleanValue().map(e -> e ? 1.0 : 0).orElse(null);
+            case STRING:
+                try {
+                    return Double.parseDouble(entry.getStrValue().orElse(""));
+                } catch (RuntimeException e) {
+                    return null;
+                }
+            case JSON:
+                try {
+                    return Double.parseDouble(entry.getJsonValue().orElse(""));
+                } catch (RuntimeException e) {
+                    return null;
+                }
+            default:
+                return null;
+        }
+    }
+
+    public static Boolean getBoolValue(KvEntry entry) {
+        switch (entry.getDataType()) {
+            case LONG:
+                return entry.getLongValue().map(e -> e != 0).orElse(null);
+            case DOUBLE:
+                return entry.getDoubleValue().map(e -> e != 0).orElse(null);
+            case BOOLEAN:
+                return entry.getBooleanValue().orElse(null);
+            case STRING:
+                try {
+                    return Boolean.parseBoolean(entry.getStrValue().orElse(""));
+                } catch (RuntimeException e) {
+                    return null;
+                }
+            case JSON:
+                try {
+                    return Boolean.parseBoolean(entry.getJsonValue().orElse(""));
+                } catch (RuntimeException e) {
+                    return null;
+                }
+            default:
+                return null;
+        }
+    }
+
+}