UI: Improvement autocomplete data keys in datasource widget

application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java

@@ -204,8 +204,8 @@ public class DefaultEntityQueryService implements EntityQueryService {
     private void replyWithResponse(DeferredResult<ResponseEntity> response, Set<EntityType> types, List<String> timeseriesKeys, List<String> attributesKeys) {
         ObjectNode json = JacksonUtil.newObjectNode();
         addItemsToArrayNode(json.putArray("entityTypes"), types);
-        addItemsToArrayNode(json.putArray("timeseriesKeys"), timeseriesKeys);
-        addItemsToArrayNode(json.putArray("attributesKeys"), attributesKeys);
+        addItemsToArrayNode(json.putArray("timeseries"), timeseriesKeys);
+        addItemsToArrayNode(json.putArray("attribute"), attributesKeys);
         response.setResult(new ResponseEntity(json, HttpStatus.OK));
     }
 

ui-ngx/src/app/core/http/entity.service.ts

@@ -41,10 +41,16 @@ import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.
 import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils';
 import { RuleChainService } from '@core/http/rule-chain.service';
 import { AliasInfo, StateParams, SubscriptionInfo } from '@core/api/widget-api.models';
-import { Datasource, DatasourceType, KeyInfo } from '@app/shared/models/widget.models';
+import { DataKey, Datasource, DatasourceType, KeyInfo } from '@app/shared/models/widget.models';
 import { UtilsService } from '@core/services/utils.service';
 import { AliasFilterType, EntityAlias, EntityAliasFilter, EntityAliasFilterResult } from '@shared/models/alias.models';
-import { entityFields, EntityInfo, ImportEntitiesResultInfo, ImportEntityData } from '@shared/models/entity.models';
+import {
+  EntitiesKeysByQuery,
+  entityFields,
+  EntityInfo,
+  ImportEntitiesResultInfo,
+  ImportEntityData
+} from '@shared/models/entity.models';
 import { EntityRelationService } from '@core/http/entity-relation.service';
 import { deepClone, isDefined, isDefinedAndNotNull } from '@core/utils';
 import { Asset } from '@shared/models/asset.models';
@@ -376,6 +382,13 @@ export class EntityService {
     return this.http.post<PageData<EntityData>>('/api/entitiesQuery/find', query, defaultHttpOptionsFromConfig(config));
   }
 
+  public findEntityKeysByQuery(query: EntityDataQuery, attributes = true, timeseries = true,
+                               config?: RequestConfig): Observable<EntitiesKeysByQuery> {
+    return this.http.post<EntitiesKeysByQuery>(
+      `/api/entitiesQuery/find/keys?attributes=${attributes}&timeseries=${timeseries}`,
+      query, defaultHttpOptionsFromConfig(config));
+  }
+
   public findAlarmDataByQuery(query: AlarmDataQuery, config?: RequestConfig): Observable<PageData<AlarmData>> {
     return this.http.post<PageData<AlarmData>>('/api/alarmsQuery/find', query, defaultHttpOptionsFromConfig(config));
   }
@@ -595,7 +608,7 @@ export class EntityService {
     return entityTypes;
   }
 
-  private getEntityFieldKeys(entityType: EntityType, searchText: string): Array<string> {
+  private getEntityFieldKeys(entityType: EntityType, searchText: string = ''): Array<string> {
     const entityFieldKeys: string[] = [entityFields.createdTime.keyName];
     const query = searchText.toLowerCase();
     switch (entityType) {
@@ -637,7 +650,7 @@ export class EntityService {
     return query ? entityFieldKeys.filter((entityField) => entityField.toLowerCase().indexOf(query) === 0) : entityFieldKeys;
   }
 
-  private getAlarmKeys(searchText: string): Array<string> {
+  private getAlarmKeys(searchText: string = ''): Array<string> {
     const alarmKeys: string[] = Object.keys(alarmFields);
     const query = searchText.toLowerCase();
     return query ? alarmKeys.filter((alarmField) => alarmField.toLowerCase().indexOf(query) === 0) : alarmKeys;
@@ -672,6 +685,59 @@ export class EntityService {
     );
   }
 
+  public getEntityKeysByEntityFilter(filter: EntityFilter, types: DataKeyType[], config?: RequestConfig): Observable<Array<DataKey>> {
+    if (!types.length) {
+      return of([]);
+    }
+    let entitiesKeysByQuery$: Observable<EntitiesKeysByQuery>;
+    if (filter !== null && types.some(type => [DataKeyType.timeseries, DataKeyType.attribute].includes(type))) {
+      const dataQuery = {
+        entityFilter: filter,
+        pageLink: createDefaultEntityDataPageLink(100),
+      };
+      entitiesKeysByQuery$ = this.findEntityKeysByQuery(dataQuery, types.includes(DataKeyType.attribute),
+        types.includes(DataKeyType.timeseries), config);
+    } else {
+      entitiesKeysByQuery$ = of({
+        attribute: [],
+        timeseries: [],
+        entityTypes: [],
+      });
+    }
+    return entitiesKeysByQuery$.pipe(
+      map((entitiesKeys) => {
+        const dataKeys: Array<DataKey> = [];
+        types.forEach(type => {
+          let keys: Array<string>;
+          switch (type) {
+            case DataKeyType.entityField:
+              if (entitiesKeys.entityTypes.length) {
+                const entitiesFields = [];
+                entitiesKeys.entityTypes.forEach(entityType => entitiesFields.push(...this.getEntityFieldKeys(entityType)));
+                keys = Array.from(new Set(entitiesFields));
+              }
+              break;
+            case DataKeyType.alarm:
+              keys = this.getAlarmKeys();
+              break;
+            case DataKeyType.attribute:
+            case DataKeyType.timeseries:
+              if (entitiesKeys[type].length) {
+                keys = entitiesKeys[type];
+              }
+              break;
+          }
+          if (keys) {
+            dataKeys.push(...keys.map(key => {
+              return {name: key, type};
+            }));
+          }
+        });
+        return dataKeys;
+      })
+    );
+  }
+
   public createDatasourcesFromSubscriptionsInfo(subscriptionsInfo: Array<SubscriptionInfo>): Array<Datasource> {
     const datasources = subscriptionsInfo.map(subscriptionInfo => this.createDatasourceFromSubscriptionInfo(subscriptionInfo));
     this.utils.generateColors(datasources);

ui-ngx/src/app/modules/home/components/widget/data-key-config.component.ts

@@ -36,7 +36,7 @@ import { EntityService } from '@core/http/entity.service';
 import { DataKeysCallbacks } from '@home/components/widget/data-keys.component.models';
 import { DataKeyType } from '@shared/models/telemetry/telemetry.models';
 import { Observable, of } from 'rxjs';
-import { map, mergeMap, tap } from 'rxjs/operators';
+import { map, mergeMap, publishReplay, refCount, tap } from 'rxjs/operators';
 import { alarmFields } from '@shared/models/alarm.models';
 import { JsFuncComponent } from '@shared/components/js-func.component';
 import { JsonFormComponentData } from '@shared/components/json-form/json-form-component.models';
@@ -95,6 +95,7 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con
 
   filteredKeys: Observable<Array<string>>;
   private latestKeySearchResult: Array<string> = null;
+  private fetchObservable$: Observable<Array<string>> = null;
 
   keySearchText = '';
 
@@ -205,31 +206,42 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con
   }
 
   private fetchKeys(searchText?: string): Observable<Array<string>> {
-    if (this.latestKeySearchResult === null || this.keySearchText !== searchText) {
+    if (this.keySearchText !== searchText || this.latestKeySearchResult === null) {
       this.keySearchText = searchText;
-      let fetchObservable: Observable<Array<DataKey>> = null;
+      const dataKeyFilter = this.createKeyFilter(this.keySearchText);
+      return this.getKeys().pipe(
+        map(name => name.filter(dataKeyFilter)),
+        tap(res => this.latestKeySearchResult = res)
+      );
+    }
+    return of(this.latestKeySearchResult);
+  }
+
+  private getKeys() {
+    if (this.fetchObservable$ === null) {
+      let fetchObservable: Observable<Array<DataKey>>;
       if (this.modelValue.type === DataKeyType.alarm) {
-        const dataKeyFilter = this.createDataKeyFilter(this.keySearchText);
-        fetchObservable = of(this.alarmKeys.filter(dataKeyFilter));
+        fetchObservable = of(this.alarmKeys);
       } else {
         if (this.entityAliasId) {
           const dataKeyTypes = [this.modelValue.type];
-          fetchObservable = this.callbacks.fetchEntityKeys(this.entityAliasId, this.keySearchText, dataKeyTypes);
+          fetchObservable = this.callbacks.fetchEntityKeys(this.entityAliasId, dataKeyTypes);
         } else {
           fetchObservable = of([]);
         }
       }
-      return fetchObservable.pipe(
+      this.fetchObservable$ = fetchObservable.pipe(
         map((dataKeys) => dataKeys.map((dataKey) => dataKey.name)),
-        tap(res => this.latestKeySearchResult = res)
+        publishReplay(1),
+        refCount()
       );
     }
-    return of(this.latestKeySearchResult);
+    return this.fetchObservable$;
   }
 
-  private createDataKeyFilter(query: string): (key: DataKey) => boolean {
+  private createKeyFilter(query: string): (key: string) => boolean {
     const lowercaseQuery = query.toLowerCase();
-    return key => key.name.toLowerCase().indexOf(lowercaseQuery) === 0;
+    return key => key.toLowerCase().startsWith(lowercaseQuery);
   }
 
   public validateOnSubmit() {

ui-ngx/src/app/modules/home/components/widget/data-keys.component.models.ts

@@ -20,5 +20,5 @@ import { Observable } from 'rxjs';
 
 export interface DataKeysCallbacks {
   generateDataKey: (chip: any, type: DataKeyType) => DataKey;
-  fetchEntityKeys: (entityAliasId: string, query: string, types: Array<DataKeyType>) => Observable<Array<DataKey>>;
+  fetchEntityKeys: (entityAliasId: string, types: Array<DataKeyType>) => Observable<Array<DataKey>>;
 }

ui-ngx/src/app/modules/home/components/widget/data-keys.component.ts

@@ -38,7 +38,7 @@ import {
     Validators
 } from '@angular/forms';
 import { Observable, of } from 'rxjs';
-import { filter, map, mergeMap, share, tap } from 'rxjs/operators';
+import { filter, map, mergeMap, publishReplay, refCount, share, tap } from 'rxjs/operators';
 import { Store } from '@ngrx/store';
 import { AppState } from '@app/core/core.state';
 import { TranslateService } from '@ngx-translate/core';
@@ -142,6 +142,7 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, AfterVie
 
   searchText = '';
   private latestSearchTextResult: Array<DataKey> = null;
+  private fetchObservable$: Observable<Array<DataKey>> = null;
 
   private dirty = false;
 
@@ -260,6 +261,7 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, AfterVie
       if (!change.firstChange && change.currentValue !== change.previousValue) {
         if (propName === 'entityAliasId') {
           this.searchText = '';
+          this.fetchObservable$ = null;
           this.latestSearchTextResult = null;
           this.dirty = true;
         } else if (['widgetType', 'datasourceType'].includes(propName)) {
@@ -405,14 +407,24 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, AfterVie
     return key ? key.name : undefined;
   }
 
-  fetchKeys(searchText?: string): Observable<Array<DataKey>> {
-    if (this.latestSearchTextResult === null || this.searchText !== searchText) {
+  private fetchKeys(searchText?: string): Observable<Array<DataKey>> {
+    if (this.searchText !== searchText || this.latestSearchTextResult === null) {
       this.searchText = searchText;
-      let fetchObservable: Observable<Array<DataKey>> = null;
+      const dataKeyFilter = this.createDataKeyFilter(this.searchText);
+      return this.getKeys().pipe(
+        map(name => name.filter(dataKeyFilter)),
+        tap(res => this.latestSearchTextResult = res)
+      );
+    }
+    return of(this.latestSearchTextResult);
+  }
+
+  private getKeys(): Observable<Array<DataKey>> {
+    if (this.fetchObservable$ === null) {
+      let fetchObservable: Observable<Array<DataKey>>;
       if (this.datasourceType === DatasourceType.function) {
-        const dataKeyFilter = this.createDataKeyFilter(this.searchText);
         const targetKeysList = this.widgetType === widgetType.alarm ? this.alarmKeys : this.functionTypeKeys;
-        fetchObservable = of(targetKeysList.filter(dataKeyFilter));
+        fetchObservable = of(targetKeysList);
       } else {
         if (this.entityAliasId) {
           const dataKeyTypes = [DataKeyType.timeseries];
@@ -420,24 +432,25 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, AfterVie
             dataKeyTypes.push(DataKeyType.attribute);
             dataKeyTypes.push(DataKeyType.entityField);
             if (this.widgetType === widgetType.alarm) {
-                dataKeyTypes.push(DataKeyType.alarm);
+              dataKeyTypes.push(DataKeyType.alarm);
             }
           }
-          fetchObservable = this.callbacks.fetchEntityKeys(this.entityAliasId, this.searchText, dataKeyTypes);
+          fetchObservable = this.callbacks.fetchEntityKeys(this.entityAliasId, dataKeyTypes);
         } else {
           fetchObservable = of([]);
         }
       }
-      return fetchObservable.pipe(
-        tap(res => this.latestSearchTextResult = res)
+      this.fetchObservable$ = fetchObservable.pipe(
+        publishReplay(1),
+        refCount()
       );
     }
-    return of(this.latestSearchTextResult);
+    return this.fetchObservable$;
   }
 
   private createDataKeyFilter(query: string): (key: DataKey) => boolean {
     const lowercaseQuery = query.toLowerCase();
-    return key => key.name.toLowerCase().indexOf(lowercaseQuery) === 0;
+    return key => key.name.toLowerCase().startsWith(lowercaseQuery);
   }
 
   textIsNotEmpty(text: string): boolean {

ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts

@@ -54,13 +54,13 @@ import { UtilsService } from '@core/services/utils.service';
 import { DataKeyType } from '@shared/models/telemetry/telemetry.models';
 import { TranslateService } from '@ngx-translate/core';
 import { EntityType } from '@shared/models/entity-type.models';
-import { forkJoin, Observable, of, Subscription } from 'rxjs';
+import { Observable, of, Subscription } from 'rxjs';
 import { WidgetConfigCallbacks } from '@home/components/widget/widget-config.component.models';
 import {
   EntityAliasDialogComponent,
   EntityAliasDialogData
 } from '@home/components/alias/entity-alias-dialog.component';
-import { catchError, map, mergeMap, tap } from 'rxjs/operators';
+import { catchError, mergeMap, tap } from 'rxjs/operators';
 import { MatDialog } from '@angular/material/dialog';
 import { EntityService } from '@core/http/entity.service';
 import { JsonFormComponentData } from '@shared/components/json-form/json-form-component.models';
@@ -792,54 +792,16 @@ export class WidgetConfigComponent extends PageComponent implements OnInit, Cont
     );
   }
 
-  private fetchEntityKeys(entityAliasId: string, query: string, dataKeyTypes: Array<DataKeyType>): Observable<Array<DataKey>> {
-    return this.aliasController.resolveSingleEntityInfo(entityAliasId).pipe(
-      mergeMap((entity) => {
-        if (entity) {
-          const fetchEntityTasks: Array<Observable<Array<DataKey>>> = [];
-          for (const dataKeyType of dataKeyTypes) {
-            fetchEntityTasks.push(
-              this.entityService.getEntityKeys(
-                {entityType: entity.entityType, id: entity.id},
-                query,
-                dataKeyType,
-                {ignoreLoading: true, ignoreErrors: true}
-              ).pipe(
-                map((keys) => {
-                  const dataKeys: Array<DataKey> = [];
-                  for (const key of keys) {
-                    dataKeys.push({name: key, type: dataKeyType});
-                  }
-                  return dataKeys;
-                }
-                ),
-                catchError(() => of([]))
-            ));
-          }
-          return forkJoin(fetchEntityTasks).pipe(
-            map(arrayOfDataKeys => {
-              const result = new Array<DataKey>();
-              arrayOfDataKeys.forEach((dataKeyArray) => {
-                result.push(...dataKeyArray);
-              });
-              return result;
-            }
-          ));
-        } else if (dataKeyTypes.includes(DataKeyType.alarm)) {
-          return this.entityService.getEntityKeys(null, query, DataKeyType.alarm).pipe(
-            map((keys) => {
-                const dataKeys: Array<DataKey> = [];
-                for (const key of keys) {
-                  dataKeys.push({name: key, type: DataKeyType.alarm});
-                }
-                return dataKeys;
-              }
-            ),
-            catchError(() => of([]))
-          );
-        } else {
-          return of([]);
-        }
+  private fetchEntityKeys(entityAliasId: string, dataKeyTypes: Array<DataKeyType>): Observable<Array<DataKey>> {
+    return this.aliasController.getAliasInfo(entityAliasId).pipe(
+      mergeMap((aliasInfo) => {
+        return this.entityService.getEntityKeysByEntityFilter(
+          aliasInfo.entityFilter,
+          dataKeyTypes,
+          {ignoreLoading: true, ignoreErrors: true}
+        ).pipe(
+          catchError(() => of([]))
+        );
       }),
       catchError(() => of([] as Array<DataKey>))
     );

ui-ngx/src/app/shared/models/entity.models.ts

@@ -64,6 +64,12 @@ export interface EntityField {
   time?: boolean;
 }
 
+export interface EntitiesKeysByQuery {
+  attribute: Array<string>;
+  timeseries: Array<string>;
+  entityTypes: EntityType[];
+}
+
 export const entityFields: {[fieldName: string]: EntityField} = {
   createdTime: {
     keyName: 'createdTime',