Merge branch 'feature/device-info-filter' of github.com:thingsboard/thingsboard into feature/device-info-filter

application/src/main/data/upgrade/3.4.4/schema_update.sql

@@ -16,16 +16,27 @@
 
 -- DEVICE INFO VIEW START
 
-DROP VIEW IF EXISTS device_info_view CASCADE;
+DROP VIEW IF EXISTS device_info_active_attribute_view CASCADE;
-CREATE OR REPLACE VIEW device_info_view as
+CREATE OR REPLACE VIEW device_info_active_attribute_view AS
 SELECT d.*
        , c.title as customer_title
-       , c.additional_info::json->>'isPublic' as customer_is_public
+       , COALESCE((c.additional_info::json->>'isPublic')::bool, FALSE) as customer_is_public
        , d.type as device_profile_name
-       , (select bool_v from attribute_kv da where da.entity_id = d.id and da.attribute_key = 'active') as attribute_active
+       , COALESCE(da.bool_v, FALSE) as active
-       , (select bool_v from ts_kv_latest dt where dt.entity_id = d.id and dt.key = (select key_id from ts_kv_dictionary where key = 'active')) as ts_active
 FROM device d
-         LEFT JOIN customer c ON c.id = d.customer_id;
+         LEFT JOIN customer c ON c.id = d.customer_id
+         LEFT JOIN attribute_kv da ON da.entity_id = d.id and da.attribute_key = 'active';
+
+DROP VIEW IF EXISTS device_info_active_ts_view CASCADE;
+CREATE OR REPLACE VIEW device_info_active_ts_view AS
+SELECT d.*
+       , c.title as customer_title
+       , COALESCE((c.additional_info::json->>'isPublic')::bool, FALSE) as customer_is_public
+       , d.type as device_profile_name
+       , COALESCE(dt.bool_v, FALSE) as active
+FROM device d
+         LEFT JOIN customer c ON c.id = d.customer_id
+         LEFT JOIN ts_kv_latest dt ON dt.entity_id = d.id and dt.key = (select key_id from ts_kv_dictionary where key = 'active');
 
 -- DEVICE INFO VIEW END
 

application/src/main/java/org/thingsboard/server/controller/DeviceController.java

@@ -680,7 +680,7 @@ public class DeviceController extends BaseController {
     @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
     @RequestMapping(value = "/edge/{edgeId}/devices", params = {"pageSize", "page"}, method = RequestMethod.GET)
     @ResponseBody
-    public PageData<Device> getEdgeDevices(
+    public PageData<DeviceInfo> getEdgeDevices(
             @ApiParam(value = EDGE_ID_PARAM_DESCRIPTION, required = true)
             @PathVariable(EDGE_ID) String strEdgeId,
             @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@@ -704,25 +704,13 @@ public class DeviceController extends BaseController {
         EdgeId edgeId = new EdgeId(toUUID(strEdgeId));
         checkEdgeId(edgeId, Operation.READ);
         TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime);
-        PageData<Device> nonFilteredResult;
+        DeviceInfoFilter.DeviceInfoFilterBuilder filter = DeviceInfoFilter.builder();
+        filter.tenantId(tenantId);
+        filter.edgeId(edgeId);
         if (type != null && type.trim().length() > 0) {
-            nonFilteredResult = deviceService.findDevicesByTenantIdAndEdgeIdAndType(tenantId, edgeId, type, pageLink);
+            filter.type(type);
-        } else {
-            nonFilteredResult = deviceService.findDevicesByTenantIdAndEdgeId(tenantId, edgeId, pageLink);
         }
-        List<Device> filteredDevices = nonFilteredResult.getData().stream().filter(device -> {
+        return checkNotNull(deviceService.findDeviceInfosByFilter(filter.build(), pageLink));
-            try {
-                accessControlService.checkPermission(getCurrentUser(), Resource.DEVICE, Operation.READ, device.getId(), device);
-                return true;
-            } catch (ThingsboardException e) {
-                return false;
-            }
-        }).collect(Collectors.toList());
-        PageData<Device> filteredResult = new PageData<>(filteredDevices,
-                nonFilteredResult.getTotalPages(),
-                nonFilteredResult.getTotalElements(),
-                nonFilteredResult.hasNext());
-        return checkNotNull(filteredResult);
     }
 
     @ApiOperation(value = "Count devices by device profile  (countByDeviceProfileAndEmptyOtaPackage)",

application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@@ -53,6 +53,9 @@ public class ThingsboardInstallService {
     @Value("${install.load_demo:false}")
     private Boolean loadDemo;
 
+    @Value("${state.persistToTelemetry:false}")
+    private boolean persistToTelemetry;
+
     @Autowired
     private EntityDatabaseSchemaService entityDatabaseSchemaService;
 
@@ -247,6 +250,7 @@ public class ThingsboardInstallService {
                         case "3.4.4":
                             log.info("Upgrading ThingsBoard from version 3.4.4 to 3.5.0 ...");
                             databaseEntitiesUpgradeService.upgradeDatabase("3.4.4");
+                            entityDatabaseSchemaService.createOrUpdateDeviceInfoView(persistToTelemetry);
                             log.info("Updating system data...");
                             systemDataLoaderService.updateSystemWidgets();
                             if (!getEnv("SKIP_DEFAULT_NOTIFICATION_CONFIGS_CREATION", false)) {
@@ -272,6 +276,8 @@ public class ThingsboardInstallService {
 
                 entityDatabaseSchemaService.createDatabaseSchema();
 
+                entityDatabaseSchemaService.createOrUpdateDeviceInfoView(persistToTelemetry);
+
                 log.info("Installing DataBase schema for timeseries...");
 
                 if (noSqlKeyspaceService != null) {

application/src/main/java/org/thingsboard/server/service/install/EntityDatabaseSchemaService.java

@@ -16,4 +16,7 @@
 package org.thingsboard.server.service.install;
 
 public interface EntityDatabaseSchemaService extends DatabaseSchemaService {
+
+    void createOrUpdateDeviceInfoView(boolean activityStateInTelemetry);
+
 }

application/src/main/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaService.java

@@ -39,4 +39,10 @@ public class SqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaSer
         executeQueryFromFile(SCHEMA_ENTITIES_IDX_PSQL_ADDON_SQL);
     }
 
+    @Override
+    public void createOrUpdateDeviceInfoView(boolean activityStateInTelemetry) {
+        String sourceViewName = activityStateInTelemetry ? "device_info_active_ts_view" : "device_info_active_attribute_view";
+        executeQuery("DROP VIEW IF EXISTS device_info_view CASCADE;");
+        executeQuery("CREATE OR REPLACE VIEW device_info_view AS SELECT * FROM " + sourceViewName + ";");
+    }
 }

application/src/main/resources/thingsboard.yml

@@ -645,6 +645,10 @@ state:
   # Should be greater then transport.sessions.report_timeout
   defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:600}"
   defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:60}"
+  # Controls whether we store device 'active' flag in attributes (default) or telemetry.
+  # If you device to change this parameter, you should re-create the device info view as one of the following:
+  # If 'persistToTelemetry' is changed from 'false' to 'true': 'CREATE OR REPLACE VIEW device_info_view AS SELECT * FROM device_info_active_ts_view;'
+  # If 'persistToTelemetry' is changed from 'true' to 'false': 'CREATE OR REPLACE VIEW device_info_view AS SELECT * FROM device_info_active_attribute_view;'
   persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}"
 
 tbel:

application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java

@@ -38,6 +38,7 @@ import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.common.util.ThingsBoardExecutors;
 import org.thingsboard.server.common.data.Customer;
 import org.thingsboard.server.common.data.Device;
+import org.thingsboard.server.common.data.DeviceInfo;
 import org.thingsboard.server.common.data.DeviceProfile;
 import org.thingsboard.server.common.data.EntitySubtype;
 import org.thingsboard.server.common.data.EntityType;
@@ -1287,8 +1288,8 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest {
                 savedDevice.getId().getId().toString(), savedEdge.getId().getId().toString(), savedEdge.getName());
         testNotificationUpdateGatewayNever();
 
-        pageData = doGetTypedWithPageLink("/api/edge/" + savedEdge.getId().getId() + "/devices?",
+        PageData<DeviceInfo> pageData = doGetTypedWithPageLink("/api/edge/" + savedEdge.getId().getId() + "/devices?",
-                PAGE_DATA_DEVICE_TYPE_REF, new PageLink(100));
+                new TypeReference<>() {}, new PageLink(100));
 
         Assert.assertEquals(1, pageData.getData().size());
 
@@ -1303,7 +1304,7 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest {
         testNotificationUpdateGatewayNever();
 
         pageData = doGetTypedWithPageLink("/api/edge/" + savedEdge.getId().getId() + "/devices?",
-                PAGE_DATA_DEVICE_TYPE_REF, new PageLink(100));
+                new TypeReference<>() {}, new PageLink(100));
 
         Assert.assertEquals(0, pageData.getData().size());
     }

common/data/src/main/java/org/thingsboard/server/common/data/DeviceInfoFilter.java

@@ -19,6 +19,7 @@ import lombok.Builder;
 import lombok.Data;
 import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.DeviceProfileId;
+import org.thingsboard.server.common.data.id.EdgeId;
 import org.thingsboard.server.common.data.id.TenantId;
 
 @Data
@@ -27,6 +28,7 @@ public class DeviceInfoFilter {
 
     private TenantId tenantId;
     private CustomerId customerId;
+    private EdgeId edgeId;
     private String type;
     private DeviceProfileId deviceProfileId;
     private Boolean active;

dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java

@@ -46,12 +46,6 @@ public abstract class DaoUtil {
         return new PageData<>(data, page.getTotalPages(), page.getTotalElements(), page.hasNext());
     }
 
-    public static <T> PageData<T> toPageData(Page<? extends ToData<T>> page, Object... params) {
-        List<T> data = convertDataList(page.getContent(), params);
-        return new PageData<>(data, page.getTotalPages(), page.getTotalElements(), page.hasNext());
-    }
-
-
     public static <T> PageData<T> pageToPageData(Page<T> page) {
         return new PageData<>(page.getContent(), page.getTotalPages(), page.getTotalElements(), page.hasNext());
     }
@@ -85,19 +79,6 @@ public abstract class DaoUtil {
         return list;
     }
 
-    public static <T> List<T> convertDataList(Collection<? extends ToData<T>> toDataList, Object... params) {
-        List<T> list = Collections.emptyList();
-        if (toDataList != null && !toDataList.isEmpty()) {
-            list = new ArrayList<>();
-            for (ToData<T> object : toDataList) {
-                if (object != null) {
-                    list.add(object.toData(params));
-                }
-            }
-        }
-        return list;
-    }
-
     public static <T> T getData(ToData<T> data) {
         T object = null;
         if (data != null) {
@@ -106,15 +87,6 @@ public abstract class DaoUtil {
         return object;
     }
 
-    public static <T> T getData(ToData<T> data, Object... params) {
-        T object = null;
-        if (data != null) {
-            object = data.toData(params);
-        }
-        return object;
-    }
-
-
     public static <T> T getData(Optional<? extends ToData<T>> data) {
         T object = null;
         if (data.isPresent()) {

dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java

@@ -172,8 +172,7 @@ public class ModelConstants {
     public static final String DEVICE_CUSTOMER_TITLE_PROPERTY = "customer_title";
     public static final String DEVICE_CUSTOMER_IS_PUBLIC_PROPERTY = "customer_is_public";
     public static final String DEVICE_DEVICE_PROFILE_NAME_PROPERTY = "device_profile_name";
-    public static final String DEVICE_ATTR_ACTIVE_PROPERTY = "attribute_active";
+    public static final String DEVICE_ACTIVE_PROPERTY = "active";
-    public static final String DEVICE_TS_ACTIVE_PROPERTY = "ts_active";
 
     public static final String DEVICE_BY_TENANT_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "device_by_tenant_and_search_text";
     public static final String DEVICE_BY_TENANT_BY_TYPE_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "device_by_tenant_by_type_and_search_text";

dao/src/main/java/org/thingsboard/server/dao/model/ToData.java

@@ -29,7 +29,4 @@ public interface ToData<T> {
      */
     T toData();
 
-    default T toData(Object... params) {
-        return toData();
-    }
 }

dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceInfoEntity.java

@@ -24,8 +24,6 @@ import org.thingsboard.server.dao.model.ModelConstants;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.Table;
-import java.util.HashMap;
-import java.util.Map;
 
 @Data
 @EqualsAndHashCode(callSuper = true)
@@ -34,23 +32,14 @@ import java.util.Map;
 @Table(name = ModelConstants.DEVICE_INFO_VIEW_COLUMN_FAMILY_NAME)
 public class DeviceInfoEntity extends AbstractDeviceEntity<DeviceInfo> {
 
-    public static final Map<String, String> attrActiveColumnMap = new HashMap<>();
-    public static final Map<String, String> tsActiveColumnMap = new HashMap<>();
-    static {
-        attrActiveColumnMap.put("active", "attributeActive");
-        tsActiveColumnMap.put("active", "tsActive");
-    }
-
     @Column(name = ModelConstants.DEVICE_CUSTOMER_TITLE_PROPERTY)
     private String customerTitle;
     @Column(name = ModelConstants.DEVICE_CUSTOMER_IS_PUBLIC_PROPERTY)
-    private Boolean customerIsPublic;
+    private boolean customerIsPublic;
     @Column(name = ModelConstants.DEVICE_DEVICE_PROFILE_NAME_PROPERTY)
     private String deviceProfileName;
-    @Column(name = ModelConstants.DEVICE_ATTR_ACTIVE_PROPERTY)
+    @Column(name = ModelConstants.DEVICE_ACTIVE_PROPERTY)
-    private Boolean attributeActive;
+    private boolean active;
-    @Column(name = ModelConstants.DEVICE_TS_ACTIVE_PROPERTY)
-    private Boolean tsActive;
 
     public DeviceInfoEntity() {
         super();
@@ -59,13 +48,7 @@ public class DeviceInfoEntity extends AbstractDeviceEntity<DeviceInfo> {
 
     @Override
     public DeviceInfo toData() {
-        return toData(false);
+        return new DeviceInfo(super.toDevice(), customerTitle, customerIsPublic, deviceProfileName, active);
     }
 
-    @Override
-    public DeviceInfo toData(Object... persistToTelemetry) {
-        boolean attr = persistToTelemetry.length == 0 || !(Boolean) persistToTelemetry[0];
-        return new DeviceInfo(super.toDevice(), customerTitle, Boolean.TRUE.equals(customerIsPublic), deviceProfileName,
-                attr ? Boolean.TRUE.equals(attributeActive) : Boolean.TRUE.equals(tsActive));
-    }
 }

dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java

@@ -21,10 +21,6 @@ import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.query.Param;
 import org.thingsboard.server.common.data.DeviceTransportType;
-import org.thingsboard.server.common.data.DeviceIdInfo;
-import org.thingsboard.server.common.data.id.CustomerId;
-import org.thingsboard.server.common.data.id.DeviceProfileId;
-import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.dao.ExportableEntityRepository;
 import org.thingsboard.server.dao.model.sql.DeviceEntity;
 import org.thingsboard.server.dao.model.sql.DeviceInfoEntity;
@@ -123,19 +119,20 @@ public interface DeviceRepository extends JpaRepository<DeviceEntity, UUID>, Exp
     @Query("SELECT d FROM DeviceInfoEntity d " +
             "WHERE d.tenantId = :tenantId " +
             "AND (:customerId IS NULL OR d.customerId = uuid(:customerId)) " +
+            "AND (:edgeId IS NULL OR d.id IN (SELECT re.toId FROM RelationEntity re WHERE re.toType = 'DEVICE' AND re.relationTypeGroup = 'EDGE' AND re.relationType = 'Contains' AND re.fromType = 'EDGE' AND re.fromId = uuid(:edgeId))) " +
             "AND ((:deviceType) IS NULL OR d.type = :deviceType) " +
             "AND (:deviceProfileId IS NULL OR d.deviceProfileId = uuid(:deviceProfileId)) " +
-            "AND ((:filterByActive) IS FALSE OR (((:persistToTelemetry) IS TRUE AND d.tsActive = :deviceActive) OR ((:persistToTelemetry) IS FALSE AND d.attributeActive = :deviceActive))) " +
+            "AND ((:filterByActive) IS FALSE OR d.active = :deviceActive) " +
             "AND (LOWER(d.searchText) LIKE LOWER(CONCAT('%', :textSearch, '%')) " +
             "OR LOWER(d.label) LIKE LOWER(CONCAT('%', :textSearch, '%')) " +
             "OR LOWER(d.type) LIKE LOWER(CONCAT('%', :textSearch, '%')) " +
             "OR LOWER(d.customerTitle) LIKE LOWER(CONCAT('%', :textSearch, '%')))")
     Page<DeviceInfoEntity> findDeviceInfosByFilter(@Param("tenantId") UUID tenantId,
                                                    @Param("customerId") String customerId,
+                                                   @Param("edgeId") String edgeId,
                                                    @Param("deviceType") String type,
                                                    @Param("deviceProfileId") String deviceProfileId,
                                                    @Param("filterByActive") boolean filterByActive,
-                                                   @Param("persistToTelemetry") boolean persistToTelemetry,
                                                    @Param("deviceActive") boolean active,
                                                    @Param("textSearch") String textSearch,
                                                    Pageable pageable);
@@ -178,7 +175,7 @@ public interface DeviceRepository extends JpaRepository<DeviceEntity, UUID>, Exp
     /**
      * Count devices by tenantId.
      * Custom query applied because default QueryDSL produces slow count(id).
-     * <p>
+     *
      * There is two way to count devices.
      * OPTIMAL: count(*)
      * - returns _row_count_ and use index-only scan (super fast).

dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java

@@ -64,11 +64,6 @@ import java.util.UUID;
 @Slf4j
 public class JpaDeviceDao extends JpaAbstractSearchTextDao<DeviceEntity, Device> implements DeviceDao {
 
-    @Value("${state.persistToTelemetry:false}")
-    private boolean persistToTelemetry;
-
-    private Map<String, String> activeColumnMap;
-
     @Autowired
     private DeviceRepository deviceRepository;
 
@@ -85,14 +80,9 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao<DeviceEntity, Device>
         return deviceRepository;
     }
 
-    @PostConstruct
-    public void init() {
-        activeColumnMap = persistToTelemetry ? DeviceInfoEntity.tsActiveColumnMap : DeviceInfoEntity.attrActiveColumnMap;
-    }
-
     @Override
     public DeviceInfo findDeviceInfoById(TenantId tenantId, UUID deviceId) {
-        return DaoUtil.getData(deviceRepository.findDeviceInfoById(deviceId), persistToTelemetry);
+        return DaoUtil.getData(deviceRepository.findDeviceInfoById(deviceId));
     }
 
     @Override
@@ -125,13 +115,13 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao<DeviceEntity, Device>
                 deviceRepository.findDeviceInfosByFilter(
                         filter.getTenantId().getId(),
                         DaoUtil.getStringId(filter.getCustomerId()),
+                        DaoUtil.getStringId(filter.getEdgeId()),
                         filter.getType(),
                         DaoUtil.getStringId(filter.getDeviceProfileId()),
                         filter.getActive() != null,
-                        persistToTelemetry,
                         Boolean.TRUE.equals(filter.getActive()),
                         Objects.toString(pageLink.getTextSearch(), ""),
-                        DaoUtil.toPageable(pageLink, activeColumnMap)), persistToTelemetry);
+                        DaoUtil.toPageable(pageLink)));
     }
 
     @Override

dao/src/main/resources/sql/schema-entities.sql

@@ -859,15 +859,30 @@ CREATE TABLE IF NOT EXISTS user_settings (
     CONSTRAINT user_settings_pkey PRIMARY KEY (user_id, type)
 );
 
-CREATE OR REPLACE VIEW device_info_view as
+DROP VIEW IF EXISTS device_info_active_attribute_view CASCADE;
+CREATE OR REPLACE VIEW device_info_active_attribute_view AS
 SELECT d.*
        , c.title as customer_title
-       , c.additional_info::json->>'isPublic' as customer_is_public
+       , COALESCE((c.additional_info::json->>'isPublic')::bool, FALSE) as customer_is_public
        , d.type as device_profile_name
-       , (select bool_v from attribute_kv da where da.entity_id = d.id and da.attribute_key = 'active') as attribute_active
+       , COALESCE(da.bool_v, FALSE) as active
-       , (select bool_v from ts_kv_latest dt where dt.entity_id = d.id and dt.key = (select key_id from ts_kv_dictionary where key = 'active')) as ts_active
 FROM device d
-         LEFT JOIN customer c ON c.id = d.customer_id;
+         LEFT JOIN customer c ON c.id = d.customer_id
+         LEFT JOIN attribute_kv da ON da.entity_id = d.id and da.attribute_key = 'active';
+
+DROP VIEW IF EXISTS device_info_active_ts_view CASCADE;
+CREATE OR REPLACE VIEW device_info_active_ts_view AS
+SELECT d.*
+       , c.title as customer_title
+       , COALESCE((c.additional_info::json->>'isPublic')::bool, FALSE) as customer_is_public
+       , d.type as device_profile_name
+       , COALESCE(dt.bool_v, FALSE) as active
+FROM device d
+         LEFT JOIN customer c ON c.id = d.customer_id
+         LEFT JOIN ts_kv_latest dt ON dt.entity_id = d.id and dt.key = (select key_id from ts_kv_dictionary where key = 'active');
+
+DROP VIEW IF EXISTS device_info_view CASCADE;
+CREATE OR REPLACE VIEW device_info_view AS SELECT * FROM device_info_active_attribute_view;
 
 DROP VIEW IF EXISTS alarm_info CASCADE;
 CREATE VIEW alarm_info AS