Merge pull request #3834 from YevhenBondarenko/feature/keys-by-query

added ability to get attributes and timeseries keys by entity query

application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java

@@ -16,18 +16,23 @@
 package org.thingsboard.server.controller;
 
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.ResponseEntity;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.context.request.async.DeferredResult;
 import org.thingsboard.server.common.data.exception.ThingsboardException;
+import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.page.PageData;
 import org.thingsboard.server.common.data.query.AlarmData;
 import org.thingsboard.server.common.data.query.AlarmDataQuery;
 import org.thingsboard.server.common.data.query.EntityCountQuery;
 import org.thingsboard.server.common.data.query.EntityData;
+import org.thingsboard.server.common.data.query.EntityDataPageLink;
 import org.thingsboard.server.common.data.query.EntityDataQuery;
 import org.thingsboard.server.queue.util.TbCoreComponent;
 import org.thingsboard.server.service.query.EntityQueryService;
@@ -40,6 +45,7 @@ public class EntityQueryController extends BaseController {
     @Autowired
     private EntityQueryService entityQueryService;
 
+    private static final int MAX_PAGE_SIZE = 100;
 
     @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
     @RequestMapping(value = "/entitiesQuery/count", method = RequestMethod.POST)
@@ -76,4 +82,24 @@ public class EntityQueryController extends BaseController {
             throw handleException(e);
         }
     }
+
+    @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
+    @RequestMapping(value = "/entitiesQuery/find/keys", method = RequestMethod.POST)
+    @ResponseBody
+    public DeferredResult<ResponseEntity> findEntityTimeseriesAndAttributesKeysByQuery(@RequestBody EntityDataQuery query,
+                                                                                       @RequestParam("timeseries") boolean isTimeseries,
+                                                                                       @RequestParam("attributes") boolean isAttributes) throws ThingsboardException {
+        TenantId tenantId = getTenantId();
+        checkNotNull(query);
+        try {
+            EntityDataPageLink pageLink = query.getPageLink();
+            if (pageLink.getPageSize() > MAX_PAGE_SIZE) {
+                pageLink.setPageSize(MAX_PAGE_SIZE);
+            }
+            return entityQueryService.getKeysByQuery(getCurrentUser(), tenantId, query, isTimeseries, isAttributes);
+        } catch (Exception e) {
+            throw handleException(e);
+        }
+    }
+
 }

application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java

@@ -15,11 +15,23 @@
  */
 package org.thingsboard.server.service.query;
 
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
 import lombok.extern.slf4j.Slf4j;
+import org.checkerframework.checker.nullness.qual.Nullable;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+import org.springframework.web.context.request.async.DeferredResult;
+import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.page.PageData;
 import org.thingsboard.server.common.data.query.AlarmData;
 import org.thingsboard.server.common.data.query.AlarmDataQuery;
@@ -31,12 +43,25 @@ import org.thingsboard.server.common.data.query.EntityDataSortOrder;
 import org.thingsboard.server.common.data.query.EntityKey;
 import org.thingsboard.server.common.data.query.EntityKeyType;
 import org.thingsboard.server.dao.alarm.AlarmService;
+import org.thingsboard.server.dao.attributes.AttributesService;
 import org.thingsboard.server.dao.entity.EntityService;
 import org.thingsboard.server.dao.model.ModelConstants;
+import org.thingsboard.server.dao.timeseries.TimeseriesService;
+import org.thingsboard.server.dao.util.mapping.JacksonUtil;
 import org.thingsboard.server.queue.util.TbCoreComponent;
+import org.thingsboard.server.service.executors.DbCallbackExecutorService;
+import org.thingsboard.server.service.security.AccessValidator;
 import org.thingsboard.server.service.security.model.SecurityUser;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
 
 @Service
 @Slf4j
@@ -52,6 +77,15 @@ public class DefaultEntityQueryService implements EntityQueryService {
     @Value("${server.ws.max_entities_per_alarm_subscription:1000}")
     private int maxEntitiesPerAlarmSubscription;
 
+    @Autowired
+    private DbCallbackExecutorService dbCallbackExecutor;
+
+    @Autowired
+    private TimeseriesService timeseriesService;
+
+    @Autowired
+    private AttributesService attributesService;
+
     @Override
     public long countEntitiesByQuery(SecurityUser securityUser, EntityCountQuery query) {
         return entityService.countEntitiesByQuery(securityUser.getTenantId(), securityUser.getCustomerId(), query);
@@ -100,4 +134,103 @@ public class DefaultEntityQueryService implements EntityQueryService {
         EntityDataPageLink edpl = new EntityDataPageLink(maxEntitiesPerAlarmSubscription, 0, null, entitiesSortOrder);
         return new EntityDataQuery(query.getEntityFilter(), edpl, query.getEntityFields(), query.getLatestValues(), query.getKeyFilters());
     }
+
+    @Override
+    public DeferredResult<ResponseEntity> getKeysByQuery(SecurityUser securityUser, TenantId tenantId, EntityDataQuery query,
+                                                         boolean isTimeseries, boolean isAttributes) {
+        final DeferredResult<ResponseEntity> response = new DeferredResult<>();
+        if (!isAttributes && !isTimeseries) {
+            replyWithEmptyResponse(response);
+            return response;
+        }
+
+        List<EntityId> ids = this.findEntityDataByQuery(securityUser, query).getData().stream()
+                .map(EntityData::getEntityId)
+                .collect(Collectors.toList());
+        if (ids.isEmpty()) {
+            replyWithEmptyResponse(response);
+            return response;
+        }
+
+        Set<EntityType> types = ids.stream().map(EntityId::getEntityType).collect(Collectors.toSet());
+        final ListenableFuture<List<String>> timeseriesKeysFuture;
+        final ListenableFuture<List<String>> attributesKeysFuture;
+
+        if (isTimeseries) {
+            timeseriesKeysFuture = dbCallbackExecutor.submit(() -> timeseriesService.findAllKeysByEntityIds(tenantId, ids));
+        } else {
+            timeseriesKeysFuture = null;
+        }
+
+        if (isAttributes) {
+            Map<EntityType, List<EntityId>> typesMap = ids.stream().collect(Collectors.groupingBy(EntityId::getEntityType));
+            List<ListenableFuture<List<String>>> futures = new ArrayList<>(typesMap.size());
+            typesMap.forEach((type, entityIds) -> futures.add(dbCallbackExecutor.submit(() -> attributesService.findAllKeysByEntityIds(tenantId, type, entityIds))));
+            attributesKeysFuture = Futures.transform(Futures.allAsList(futures), lists -> {
+                if (CollectionUtils.isEmpty(lists)) {
+                    return Collections.emptyList();
+                }
+                return lists.stream().flatMap(List::stream).distinct().sorted().collect(Collectors.toList());
+            }, dbCallbackExecutor);
+        } else {
+            attributesKeysFuture = null;
+        }
+
+        if (isTimeseries && isAttributes) {
+            Futures.whenAllComplete(timeseriesKeysFuture, attributesKeysFuture).run(() -> {
+                try {
+                    replyWithResponse(response, types, timeseriesKeysFuture.get(), attributesKeysFuture.get());
+                } catch (Exception e) {
+                    log.error("Failed to fetch timeseries and attributes keys!", e);
+                    AccessValidator.handleError(e, response, HttpStatus.INTERNAL_SERVER_ERROR);
+                }
+            }, dbCallbackExecutor);
+        } else if (isTimeseries) {
+            addCallback(timeseriesKeysFuture, keys -> replyWithResponse(response, types, keys, null),
+                    error -> {
+                        log.error("Failed to fetch timeseries keys!", error);
+                        AccessValidator.handleError(error, response, HttpStatus.INTERNAL_SERVER_ERROR);
+                    });
+        } else {
+            addCallback(attributesKeysFuture, keys -> replyWithResponse(response, types, null, keys),
+                    error -> {
+                        log.error("Failed to fetch attributes keys!", error);
+                        AccessValidator.handleError(error, response, HttpStatus.INTERNAL_SERVER_ERROR);
+                    });
+        }
+        return response;
+    }
+
+    private void replyWithResponse(DeferredResult<ResponseEntity> response, Set<EntityType> types, List<String> timeseriesKeys, List<String> attributesKeys) {
+        ObjectNode json = JacksonUtil.newObjectNode();
+        addItemsToArrayNode(json.putArray("entityTypes"), types);
+        addItemsToArrayNode(json.putArray("timeseries"), timeseriesKeys);
+        addItemsToArrayNode(json.putArray("attribute"), attributesKeys);
+        response.setResult(new ResponseEntity(json, HttpStatus.OK));
+    }
+
+    private void replyWithEmptyResponse(DeferredResult<ResponseEntity> response) {
+        replyWithResponse(response, Collections.emptySet(), Collections.emptyList(), Collections.emptyList());
+    }
+
+    private void addItemsToArrayNode(ArrayNode arrayNode, Collection<?> collection) {
+        if (!CollectionUtils.isEmpty(collection)) {
+            collection.forEach(item -> arrayNode.add(item.toString()));
+        }
+    }
+
+    private void addCallback(ListenableFuture<List<String>> future, Consumer<List<String>> success, Consumer<Throwable> error) {
+        Futures.addCallback(future, new FutureCallback<List<String>>() {
+            @Override
+            public void onSuccess(@Nullable List<String> keys) {
+                success.accept(keys);
+            }
+
+            @Override
+            public void onFailure(Throwable t) {
+                error.accept(t);
+            }
+        }, dbCallbackExecutor);
+    }
+
 }

application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java

@@ -15,6 +15,9 @@
  */
 package org.thingsboard.server.service.query;
 
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.context.request.async.DeferredResult;
+import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.page.PageData;
 import org.thingsboard.server.common.data.query.AlarmData;
 import org.thingsboard.server.common.data.query.AlarmDataQuery;
@@ -31,4 +34,7 @@ public interface EntityQueryService {
 
     PageData<AlarmData> findAlarmDataByQuery(SecurityUser securityUser, AlarmDataQuery query);
 
+    DeferredResult<ResponseEntity> getKeysByQuery(SecurityUser securityUser, TenantId tenantId, EntityDataQuery query,
+                                                  boolean isTimeseries, boolean isAttributes);
+
 }

common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java

@@ -16,6 +16,7 @@
 package org.thingsboard.server.dao.attributes;
 
 import com.google.common.util.concurrent.ListenableFuture;
+import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.id.DeviceProfileId;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
@@ -42,4 +43,6 @@ public interface AttributesService {
 
     List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId);
 
+    List<String> findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List<EntityId> entityIds);
+
 }

common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java

@@ -50,4 +50,6 @@ public interface TimeseriesService {
     ListenableFuture<Collection<String>> removeAllLatest(TenantId tenantId, EntityId entityId);
 
     List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId);
+
+    List<String> findAllKeysByEntityIds(TenantId tenantId, List<EntityId> entityIds);
 }

dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java

@@ -16,6 +16,7 @@
 package org.thingsboard.server.dao.attributes;
 
 import com.google.common.util.concurrent.ListenableFuture;
+import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.id.DeviceProfileId;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
@@ -41,4 +42,6 @@ public interface AttributesDao {
     ListenableFuture<List<Void>> removeAll(TenantId tenantId, EntityId entityId, String attributeType, List<String> keys);
 
     List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId);
+
+    List<String> findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List<EntityId> entityIds);
 }

dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java

@@ -20,6 +20,7 @@ import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
+import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.id.DeviceProfileId;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
@@ -65,6 +66,11 @@ public class BaseAttributesService implements AttributesService {
         return attributesDao.findAllKeysByDeviceProfileId(tenantId, deviceProfileId);
     }
 
+    @Override
+    public List<String> findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List<EntityId> entityIds) {
+        return attributesDao.findAllKeysByEntityIds(tenantId, entityType, entityIds);
+    }
+
     @Override
     public ListenableFuture<List<Void>> save(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes) {
         validate(entityId, scope);

dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java

@@ -56,5 +56,8 @@ public interface AttributeKvRepository extends CrudRepository<AttributeKvEntity,
             "AND entity_id in (SELECT id FROM device WHERE tenant_id = :tenantId limit 100) ORDER BY attribute_key", nativeQuery = true)
     List<String> findAllKeysByTenantId(@Param("tenantId") UUID tenantId);
 
+    @Query(value = "SELECT DISTINCT attribute_key FROM attribute_kv WHERE entity_type = :entityType " +
+            "AND entity_id in :entityIds ORDER BY attribute_key", nativeQuery = true)
+    List<String> findAllKeysByEntityIds(@Param("entityType") String entityType, @Param("entityIds") List<UUID> entityIds);
 }
 

dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java

@@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
+import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.id.DeviceProfileId;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
@@ -145,6 +146,12 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl
         }
     }
 
+    @Override
+    public List<String> findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List<EntityId> entityIds) {
+        return attributeKvRepository
+                .findAllKeysByEntityIds(entityType.name(), entityIds.stream().map(EntityId::getId).collect(Collectors.toList()));
+    }
+
     @Override
     public ListenableFuture<Void> save(TenantId tenantId, EntityId entityId, String attributeType, AttributeKvEntry attribute) {
         AttributeKvEntity entity = new AttributeKvEntity();

dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java

@@ -61,6 +61,7 @@ import java.util.Optional;
 import java.util.UUID;
 import java.util.concurrent.ExecutionException;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 @Slf4j
 @Component
@@ -169,6 +170,11 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme
         }
     }
 
+    @Override
+    public List<String> findAllKeysByEntityIds(TenantId tenantId, List<EntityId> entityIds) {
+        return tsKvLatestRepository.findAllKeysByEntityIds(entityIds.stream().map(EntityId::getId).collect(Collectors.toList()));
+    }
+
     private ListenableFuture<Void> getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
         ListenableFuture<List<TsKvEntry>> future = findNewLatestEntryFuture(tenantId, entityId, query);
         return Futures.transformAsync(future, entryList -> {

dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java

@@ -36,4 +36,9 @@ public interface TsKvLatestRepository extends CrudRepository<TsKvLatestEntity, T
             "WHERE ts_kv_latest.entity_id IN (SELECT id FROM device WHERE tenant_id = :tenant_id limit 100) ORDER BY ts_kv_dictionary.key", nativeQuery = true)
     List<String> getKeysByTenantId(@Param("tenant_id") UUID tenantId);
 
+    @Query(value = "SELECT DISTINCT ts_kv_dictionary.key AS strKey FROM ts_kv_latest " +
+            "INNER JOIN ts_kv_dictionary ON ts_kv_latest.key = ts_kv_dictionary.key_id " +
+            "WHERE ts_kv_latest.entity_id IN :entityIds ORDER BY ts_kv_dictionary.key", nativeQuery = true)
+    List<String> findAllKeysByEntityIds(@Param("entityIds") List<UUID> entityIds);
+
 }

dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java

@@ -121,6 +121,11 @@ public class BaseTimeseriesService implements TimeseriesService {
         return timeseriesLatestDao.findAllKeysByDeviceProfileId(tenantId, deviceProfileId);
     }
 
+    @Override
+    public List<String> findAllKeysByEntityIds(TenantId tenantId, List<EntityId> entityIds) {
+        return timeseriesLatestDao.findAllKeysByEntityIds(tenantId, entityIds);
+    }
+
     @Override
     public ListenableFuture<Integer> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) {
         validate(entityId);

dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java

@@ -86,6 +86,11 @@ public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimes
         return Collections.emptyList();
     }
 
+    @Override
+    public List<String> findAllKeysByEntityIds(TenantId tenantId, List<EntityId> entityIds) {
+        return Collections.emptyList();
+    }
+
     @Override
     public ListenableFuture<Void> saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) {
         BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getLatestStmt().bind());

dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java

@@ -35,4 +35,6 @@ public interface TimeseriesLatestDao {
     ListenableFuture<Void> removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query);
 
     List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId);
+
+    List<String> findAllKeysByEntityIds(TenantId tenantId, List<EntityId> entityIds);
 }

ui-ngx/src/app/core/http/entity.service.ts

@@ -41,10 +41,16 @@ import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.
 import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils';
 import { RuleChainService } from '@core/http/rule-chain.service';
 import { AliasInfo, StateParams, SubscriptionInfo } from '@core/api/widget-api.models';
-import { Datasource, DatasourceType, KeyInfo } from '@app/shared/models/widget.models';
+import { DataKey, Datasource, DatasourceType, KeyInfo } from '@app/shared/models/widget.models';
 import { UtilsService } from '@core/services/utils.service';
 import { AliasFilterType, EntityAlias, EntityAliasFilter, EntityAliasFilterResult } from '@shared/models/alias.models';
-import { entityFields, EntityInfo, ImportEntitiesResultInfo, ImportEntityData } from '@shared/models/entity.models';
+import {
+  EntitiesKeysByQuery,
+  entityFields,
+  EntityInfo,
+  ImportEntitiesResultInfo,
+  ImportEntityData
+} from '@shared/models/entity.models';
 import { EntityRelationService } from '@core/http/entity-relation.service';
 import { deepClone, isDefined, isDefinedAndNotNull } from '@core/utils';
 import { Asset } from '@shared/models/asset.models';
@@ -376,6 +382,13 @@ export class EntityService {
     return this.http.post<PageData<EntityData>>('/api/entitiesQuery/find', query, defaultHttpOptionsFromConfig(config));
   }
 
+  public findEntityKeysByQuery(query: EntityDataQuery, attributes = true, timeseries = true,
+                               config?: RequestConfig): Observable<EntitiesKeysByQuery> {
+    return this.http.post<EntitiesKeysByQuery>(
+      `/api/entitiesQuery/find/keys?attributes=${attributes}&timeseries=${timeseries}`,
+      query, defaultHttpOptionsFromConfig(config));
+  }
+
   public findAlarmDataByQuery(query: AlarmDataQuery, config?: RequestConfig): Observable<PageData<AlarmData>> {
     return this.http.post<PageData<AlarmData>>('/api/alarmsQuery/find', query, defaultHttpOptionsFromConfig(config));
   }
@@ -595,7 +608,7 @@ export class EntityService {
     return entityTypes;
   }
 
-  private getEntityFieldKeys(entityType: EntityType, searchText: string): Array<string> {
+  private getEntityFieldKeys(entityType: EntityType, searchText: string = ''): Array<string> {
     const entityFieldKeys: string[] = [entityFields.createdTime.keyName];
     const query = searchText.toLowerCase();
     switch (entityType) {
@@ -637,7 +650,7 @@ export class EntityService {
     return query ? entityFieldKeys.filter((entityField) => entityField.toLowerCase().indexOf(query) === 0) : entityFieldKeys;
   }
 
-  private getAlarmKeys(searchText: string): Array<string> {
+  private getAlarmKeys(searchText: string = ''): Array<string> {
     const alarmKeys: string[] = Object.keys(alarmFields);
     const query = searchText.toLowerCase();
     return query ? alarmKeys.filter((alarmField) => alarmField.toLowerCase().indexOf(query) === 0) : alarmKeys;
@@ -672,6 +685,59 @@ export class EntityService {
     );
   }
 
+  public getEntityKeysByEntityFilter(filter: EntityFilter, types: DataKeyType[], config?: RequestConfig): Observable<Array<DataKey>> {
+    if (!types.length) {
+      return of([]);
+    }
+    let entitiesKeysByQuery$: Observable<EntitiesKeysByQuery>;
+    if (filter !== null && types.some(type => [DataKeyType.timeseries, DataKeyType.attribute].includes(type))) {
+      const dataQuery = {
+        entityFilter: filter,
+        pageLink: createDefaultEntityDataPageLink(100),
+      };
+      entitiesKeysByQuery$ = this.findEntityKeysByQuery(dataQuery, types.includes(DataKeyType.attribute),
+        types.includes(DataKeyType.timeseries), config);
+    } else {
+      entitiesKeysByQuery$ = of({
+        attribute: [],
+        timeseries: [],
+        entityTypes: [],
+      });
+    }
+    return entitiesKeysByQuery$.pipe(
+      map((entitiesKeys) => {
+        const dataKeys: Array<DataKey> = [];
+        types.forEach(type => {
+          let keys: Array<string>;
+          switch (type) {
+            case DataKeyType.entityField:
+              if (entitiesKeys.entityTypes.length) {
+                const entitiesFields = [];
+                entitiesKeys.entityTypes.forEach(entityType => entitiesFields.push(...this.getEntityFieldKeys(entityType)));
+                keys = Array.from(new Set(entitiesFields));
+              }
+              break;
+            case DataKeyType.alarm:
+              keys = this.getAlarmKeys();
+              break;
+            case DataKeyType.attribute:
+            case DataKeyType.timeseries:
+              if (entitiesKeys[type].length) {
+                keys = entitiesKeys[type];
+              }
+              break;
+          }
+          if (keys) {
+            dataKeys.push(...keys.map(key => {
+              return {name: key, type};
+            }));
+          }
+        });
+        return dataKeys;
+      })
+    );
+  }
+
   public createDatasourcesFromSubscriptionsInfo(subscriptionsInfo: Array<SubscriptionInfo>): Array<Datasource> {
     const datasources = subscriptionsInfo.map(subscriptionInfo => this.createDatasourceFromSubscriptionInfo(subscriptionInfo));
     this.utils.generateColors(datasources);

ui-ngx/src/app/modules/home/components/widget/data-key-config.component.ts

@@ -36,7 +36,7 @@ import { EntityService } from '@core/http/entity.service';
 import { DataKeysCallbacks } from '@home/components/widget/data-keys.component.models';
 import { DataKeyType } from '@shared/models/telemetry/telemetry.models';
 import { Observable, of } from 'rxjs';
-import { map, mergeMap, tap } from 'rxjs/operators';
+import { map, mergeMap, publishReplay, refCount, tap } from 'rxjs/operators';
 import { alarmFields } from '@shared/models/alarm.models';
 import { JsFuncComponent } from '@shared/components/js-func.component';
 import { JsonFormComponentData } from '@shared/components/json-form/json-form-component.models';
@@ -95,6 +95,7 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con
 
   filteredKeys: Observable<Array<string>>;
   private latestKeySearchResult: Array<string> = null;
+  private fetchObservable$: Observable<Array<string>> = null;
 
   keySearchText = '';
 
@@ -205,31 +206,42 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con
   }
 
   private fetchKeys(searchText?: string): Observable<Array<string>> {
-    if (this.latestKeySearchResult === null || this.keySearchText !== searchText) {
+    if (this.keySearchText !== searchText || this.latestKeySearchResult === null) {
       this.keySearchText = searchText;
-      let fetchObservable: Observable<Array<DataKey>> = null;
+      const dataKeyFilter = this.createKeyFilter(this.keySearchText);
+      return this.getKeys().pipe(
+        map(name => name.filter(dataKeyFilter)),
+        tap(res => this.latestKeySearchResult = res)
+      );
+    }
+    return of(this.latestKeySearchResult);
+  }
+
+  private getKeys() {
+    if (this.fetchObservable$ === null) {
+      let fetchObservable: Observable<Array<DataKey>>;
       if (this.modelValue.type === DataKeyType.alarm) {
-        const dataKeyFilter = this.createDataKeyFilter(this.keySearchText);
+        fetchObservable = of(this.alarmKeys);
-        fetchObservable = of(this.alarmKeys.filter(dataKeyFilter));
       } else {
         if (this.entityAliasId) {
           const dataKeyTypes = [this.modelValue.type];
-          fetchObservable = this.callbacks.fetchEntityKeys(this.entityAliasId, this.keySearchText, dataKeyTypes);
+          fetchObservable = this.callbacks.fetchEntityKeys(this.entityAliasId, dataKeyTypes);
         } else {
           fetchObservable = of([]);
         }
       }
-      return fetchObservable.pipe(
+      this.fetchObservable$ = fetchObservable.pipe(
         map((dataKeys) => dataKeys.map((dataKey) => dataKey.name)),
-        tap(res => this.latestKeySearchResult = res)
+        publishReplay(1),
+        refCount()
       );
     }
-    return of(this.latestKeySearchResult);
+    return this.fetchObservable$;
   }
 
-  private createDataKeyFilter(query: string): (key: DataKey) => boolean {
+  private createKeyFilter(query: string): (key: string) => boolean {
     const lowercaseQuery = query.toLowerCase();
-    return key => key.name.toLowerCase().indexOf(lowercaseQuery) === 0;
+    return key => key.toLowerCase().startsWith(lowercaseQuery);
   }
 
   public validateOnSubmit() {

ui-ngx/src/app/modules/home/components/widget/data-keys.component.models.ts

@@ -20,5 +20,5 @@ import { Observable } from 'rxjs';
 
 export interface DataKeysCallbacks {
   generateDataKey: (chip: any, type: DataKeyType) => DataKey;
-  fetchEntityKeys: (entityAliasId: string, query: string, types: Array<DataKeyType>) => Observable<Array<DataKey>>;
+  fetchEntityKeys: (entityAliasId: string, types: Array<DataKeyType>) => Observable<Array<DataKey>>;
 }

ui-ngx/src/app/modules/home/components/widget/data-keys.component.ts

@@ -38,7 +38,7 @@ import {
     Validators
 } from '@angular/forms';
 import { Observable, of } from 'rxjs';
-import { filter, map, mergeMap, share, tap } from 'rxjs/operators';
+import { filter, map, mergeMap, publishReplay, refCount, share, tap } from 'rxjs/operators';
 import { Store } from '@ngrx/store';
 import { AppState } from '@app/core/core.state';
 import { TranslateService } from '@ngx-translate/core';
@@ -142,6 +142,7 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, AfterVie
 
   searchText = '';
   private latestSearchTextResult: Array<DataKey> = null;
+  private fetchObservable$: Observable<Array<DataKey>> = null;
 
   private dirty = false;
 
@@ -260,6 +261,7 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, AfterVie
       if (!change.firstChange && change.currentValue !== change.previousValue) {
         if (propName === 'entityAliasId') {
           this.searchText = '';
+          this.fetchObservable$ = null;
           this.latestSearchTextResult = null;
           this.dirty = true;
         } else if (['widgetType', 'datasourceType'].includes(propName)) {
@@ -405,14 +407,24 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, AfterVie
     return key ? key.name : undefined;
   }
 
-  fetchKeys(searchText?: string): Observable<Array<DataKey>> {
+  private fetchKeys(searchText?: string): Observable<Array<DataKey>> {
-    if (this.latestSearchTextResult === null || this.searchText !== searchText) {
+    if (this.searchText !== searchText || this.latestSearchTextResult === null) {
       this.searchText = searchText;
-      let fetchObservable: Observable<Array<DataKey>> = null;
+      const dataKeyFilter = this.createDataKeyFilter(this.searchText);
+      return this.getKeys().pipe(
+        map(name => name.filter(dataKeyFilter)),
+        tap(res => this.latestSearchTextResult = res)
+      );
+    }
+    return of(this.latestSearchTextResult);
+  }
+
+  private getKeys(): Observable<Array<DataKey>> {
+    if (this.fetchObservable$ === null) {
+      let fetchObservable: Observable<Array<DataKey>>;
       if (this.datasourceType === DatasourceType.function) {
-        const dataKeyFilter = this.createDataKeyFilter(this.searchText);
         const targetKeysList = this.widgetType === widgetType.alarm ? this.alarmKeys : this.functionTypeKeys;
-        fetchObservable = of(targetKeysList.filter(dataKeyFilter));
+        fetchObservable = of(targetKeysList);
       } else {
         if (this.entityAliasId) {
           const dataKeyTypes = [DataKeyType.timeseries];
@@ -420,24 +432,25 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, AfterVie
             dataKeyTypes.push(DataKeyType.attribute);
             dataKeyTypes.push(DataKeyType.entityField);
             if (this.widgetType === widgetType.alarm) {
-                dataKeyTypes.push(DataKeyType.alarm);
+              dataKeyTypes.push(DataKeyType.alarm);
             }
           }
-          fetchObservable = this.callbacks.fetchEntityKeys(this.entityAliasId, this.searchText, dataKeyTypes);
+          fetchObservable = this.callbacks.fetchEntityKeys(this.entityAliasId, dataKeyTypes);
         } else {
           fetchObservable = of([]);
         }
       }
-      return fetchObservable.pipe(
+      this.fetchObservable$ = fetchObservable.pipe(
-        tap(res => this.latestSearchTextResult = res)
+        publishReplay(1),
+        refCount()
       );
     }
-    return of(this.latestSearchTextResult);
+    return this.fetchObservable$;
   }
 
   private createDataKeyFilter(query: string): (key: DataKey) => boolean {
     const lowercaseQuery = query.toLowerCase();
-    return key => key.name.toLowerCase().indexOf(lowercaseQuery) === 0;
+    return key => key.name.toLowerCase().startsWith(lowercaseQuery);
   }
 
   textIsNotEmpty(text: string): boolean {

ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts

@@ -54,13 +54,13 @@ import { UtilsService } from '@core/services/utils.service';
 import { DataKeyType } from '@shared/models/telemetry/telemetry.models';
 import { TranslateService } from '@ngx-translate/core';
 import { EntityType } from '@shared/models/entity-type.models';
-import { forkJoin, Observable, of, Subscription } from 'rxjs';
+import { Observable, of, Subscription } from 'rxjs';
 import { WidgetConfigCallbacks } from '@home/components/widget/widget-config.component.models';
 import {
   EntityAliasDialogComponent,
   EntityAliasDialogData
 } from '@home/components/alias/entity-alias-dialog.component';
-import { catchError, map, mergeMap, tap } from 'rxjs/operators';
+import { catchError, mergeMap, tap } from 'rxjs/operators';
 import { MatDialog } from '@angular/material/dialog';
 import { EntityService } from '@core/http/entity.service';
 import { JsonFormComponentData } from '@shared/components/json-form/json-form-component.models';
@@ -792,54 +792,16 @@ export class WidgetConfigComponent extends PageComponent implements OnInit, Cont
     );
   }
 
-  private fetchEntityKeys(entityAliasId: string, query: string, dataKeyTypes: Array<DataKeyType>): Observable<Array<DataKey>> {
+  private fetchEntityKeys(entityAliasId: string, dataKeyTypes: Array<DataKeyType>): Observable<Array<DataKey>> {
-    return this.aliasController.resolveSingleEntityInfo(entityAliasId).pipe(
+    return this.aliasController.getAliasInfo(entityAliasId).pipe(
-      mergeMap((entity) => {
+      mergeMap((aliasInfo) => {
-        if (entity) {
+        return this.entityService.getEntityKeysByEntityFilter(
-          const fetchEntityTasks: Array<Observable<Array<DataKey>>> = [];
+          aliasInfo.entityFilter,
-          for (const dataKeyType of dataKeyTypes) {
+          dataKeyTypes,
-            fetchEntityTasks.push(
+          {ignoreLoading: true, ignoreErrors: true}
-              this.entityService.getEntityKeys(
+        ).pipe(
-                {entityType: entity.entityType, id: entity.id},
+          catchError(() => of([]))
-                query,
+        );
-                dataKeyType,
-                {ignoreLoading: true, ignoreErrors: true}
-              ).pipe(
-                map((keys) => {
-                  const dataKeys: Array<DataKey> = [];
-                  for (const key of keys) {
-                    dataKeys.push({name: key, type: dataKeyType});
-                  }
-                  return dataKeys;
-                }
-                ),
-                catchError(() => of([]))
-            ));
-          }
-          return forkJoin(fetchEntityTasks).pipe(
-            map(arrayOfDataKeys => {
-              const result = new Array<DataKey>();
-              arrayOfDataKeys.forEach((dataKeyArray) => {
-                result.push(...dataKeyArray);
-              });
-              return result;
-            }
-          ));
-        } else if (dataKeyTypes.includes(DataKeyType.alarm)) {
-          return this.entityService.getEntityKeys(null, query, DataKeyType.alarm).pipe(
-            map((keys) => {
-                const dataKeys: Array<DataKey> = [];
-                for (const key of keys) {
-                  dataKeys.push({name: key, type: DataKeyType.alarm});
-                }
-                return dataKeys;
-              }
-            ),
-            catchError(() => of([]))
-          );
-        } else {
-          return of([]);
-        }
       }),
       catchError(() => of([] as Array<DataKey>))
     );

ui-ngx/src/app/shared/models/entity.models.ts

@@ -64,6 +64,12 @@ export interface EntityField {
   time?: boolean;
 }
 
+export interface EntitiesKeysByQuery {
+  attribute: Array<string>;
+  timeseries: Array<string>;
+  entityTypes: EntityType[];
+}
+
 export const entityFields: {[fieldName: string]: EntityField} = {
   createdTime: {
     keyName: 'createdTime',