Merge branch 'feature/entity-data-query' of github.com:thingsboard/thingsboard into feature/entity-data-query

ui-ngx/src/app/core/api/entity-data-subscription.ts

@@ -232,13 +232,7 @@ export class EntityDataSubscription {
             this.subsCommand.tsCmd.interval = this.subsTw.aggregation.interval;
             this.subsCommand.tsCmd.limit = this.subsTw.aggregation.limit;
             this.subsCommand.tsCmd.agg = this.subsTw.aggregation.type;
-            if (this.subsTw.aggregation.stateData) {
-              this.subsCommand.historyCmd.startTs = this.subsTw.startTs - YEAR;
-              this.subsCommand.historyCmd.endTs = this.subsTw.startTs;
-              this.subsCommand.historyCmd.interval = this.subsTw.aggregation.interval;
-              this.subsCommand.historyCmd.limit = this.subsTw.aggregation.limit;
-              this.subsCommand.historyCmd.agg = this.subsTw.aggregation.type;
-            }
+            this.subsCommand.tsCmd.fetchLatestPreviousPoint = this.subsTw.aggregation.stateData;
             this.dataAggregators.forEach((dataAggregator) => {
               dataAggregator.reset(newSubsTw.startTs,  newSubsTw.aggregation.timeWindow, newSubsTw.aggregation.interval);
             });
@@ -324,7 +318,7 @@ export class EntityDataSubscription {
         if (this.datasourceType === DatasourceType.function) {
           this.dataAggregators[dataIndex] = this.createRealtimeDataAggregator(this.subsTw, tsKeyNames,
             DataKeyType.function, dataIndex, this.notifyListener.bind(this));
-        } else if (!this.history && tsKeyNames.length) {
+        } else if (tsKeyNames.length) {
           this.dataAggregators[dataIndex] = this.createRealtimeDataAggregator(this.subsTw, tsKeyNames,
             DataKeyType.timeseries, dataIndex, this.notifyListener.bind(this));
         }
@@ -342,11 +336,9 @@ export class EntityDataSubscription {
               endTs: this.subsTw.fixedWindow.endTimeMs,
               interval: this.subsTw.aggregation.interval,
               limit: this.subsTw.aggregation.limit,
-              agg: this.subsTw.aggregation.type
+              agg: this.subsTw.aggregation.type,
+              fetchLatestPreviousPoint: this.subsTw.aggregation.stateData
             };
-            if (this.subsTw.aggregation.stateData) {
-              this.subsCommand.historyCmd.startTs -= YEAR;
-            }
           } else {
             this.subsCommand.tsCmd = {
               keys: this.tsFields.map(key => key.key),
@@ -354,17 +346,8 @@ export class EntityDataSubscription {
               timeWindow: this.subsTw.aggregation.timeWindow,
               interval: this.subsTw.aggregation.interval,
               limit: this.subsTw.aggregation.limit,
-              agg: this.subsTw.aggregation.type
-            }
-            if (this.subsTw.aggregation.stateData) {
-              this.subsCommand.historyCmd = {
-                keys: this.tsFields.map(key => key.key),
-                startTs: this.subsTw.startTs - YEAR,
-                endTs: this.subsTw.startTs,
-                interval: this.subsTw.aggregation.interval,
-                limit: this.subsTw.aggregation.limit,
-                agg: this.subsTw.aggregation.type
-              };
+              agg: this.subsTw.aggregation.type,
+              fetchLatestPreviousPoint: this.subsTw.aggregation.stateData
             }
           }
         }
@@ -506,14 +489,10 @@ export class EntityDataSubscription {
     }
     if (this.entityDataSubscriptionOptions.type === widgetType.timeseries && entityData.timeseries) {
       const subscriptionData = this.toSubscriptionData(entityData.timeseries, true);
-      if (!this.history) {
-        if (this.dataAggregators && this.dataAggregators[dataIndex]) {
-          this.dataAggregators[dataIndex].onData({data: subscriptionData}, false, false, true);
-        }
-        if (!aggregate) {
-          this.onData(subscriptionData, DataKeyType.timeseries, dataIndex, true, dataUpdatedCb);
-        }
-      } else {
+      if (this.dataAggregators && this.dataAggregators[dataIndex]) {
+        this.dataAggregators[dataIndex].onData({data: subscriptionData}, false, this.history, true);
+      }
+      if (!this.history && !aggregate) {
         this.onData(subscriptionData, DataKeyType.timeseries, dataIndex, true, dataUpdatedCb);
       }
     }

ui-ngx/src/app/core/api/widget-api.models.ts

@@ -44,6 +44,7 @@ import * as moment_ from 'moment';
 import { EntityData, EntityDataPageLink, EntityFilter, KeyFilter } from '@shared/models/query/query.models';
 import { EntityDataService } from '@core/api/entity-data.service';
 import { PageData } from '@shared/models/page/page-data';
+import { TranslateService } from '@ngx-translate/core';
 
 export interface TimewindowFunctions {
   onUpdateTimewindow: (startTimeMs: number, endTimeMs: number, interval?: number) => void;
@@ -174,6 +175,7 @@ export class WidgetSubscriptionContext {
   timeService: TimeService;
   deviceService: DeviceService;
   alarmService: AlarmService;
+  translate: TranslateService;
   // datasourceService: DatasourceService;
   entityDataService: EntityDataService;
   utils: UtilsService;

ui-ngx/src/app/core/api/widget-subscription.ts

@@ -22,6 +22,7 @@ import {
   WidgetSubscriptionOptions
 } from '@core/api/widget-api.models';
 import {
+  DataKey,
   DataSet,
   DataSetHolder,
   Datasource,
@@ -393,6 +394,28 @@ export class WidgetSubscription implements IWidgetSubscription {
       this.notifyDataLoaded();
       return of(null);
     }
+    if (this.comparisonEnabled) {
+      const additionalDatasources: Datasource[] = [];
+      this.configuredDatasources.forEach((datasource, datasourceIndex) => {
+        const additionalDataKeys: DataKey[] = [];
+        datasource.dataKeys.forEach((dataKey, dataKeyIndex) => {
+          if (dataKey.settings.comparisonSettings && dataKey.settings.comparisonSettings.showValuesForComparison) {
+            const additionalDataKey = deepClone(dataKey);
+            additionalDataKey.isAdditional = true;
+            additionalDataKey.origDataKeyIndex = dataKeyIndex;
+            additionalDataKeys.push(additionalDataKey);
+          }
+        });
+        if (additionalDataKeys.length) {
+          const additionalDatasource: Datasource = deepClone(datasource);
+          additionalDatasource.dataKeys = additionalDataKeys;
+          additionalDatasource.isAdditional = true;
+          additionalDatasource.origDatasourceIndex = datasourceIndex;
+          additionalDatasources.push(additionalDatasource);
+        }
+      });
+      this.configuredDatasources = this.configuredDatasources.concat(additionalDatasources);
+    }
     const resolveResultObservables = this.configuredDatasources.map((datasource, index) => {
       const listener: EntityDataListener = {
         subscriptionType: this.type,
@@ -404,11 +427,18 @@ export class WidgetSubscription implements IWidgetSubscription {
         initialPageDataChanged: this.initialPageDataChanged.bind(this),
         dataUpdated: this.dataUpdated.bind(this),
         updateRealtimeSubscription: () => {
-          this.subscriptionTimewindow = this.updateRealtimeSubscription();
-          return this.subscriptionTimewindow;
+          if (this.comparisonEnabled && datasource.isAdditional) {
+            return this.updateSubscriptionForComparison();
+          } else {
+            return this.updateRealtimeSubscription();
+          }
         },
         setRealtimeSubscription: (subscriptionTimewindow) => {
-          this.updateRealtimeSubscription(deepClone(subscriptionTimewindow));
+          if (this.comparisonEnabled && datasource.isAdditional) {
+            this.updateSubscriptionForComparison(subscriptionTimewindow);
+          } else {
+            this.updateRealtimeSubscription(deepClone(subscriptionTimewindow));
+          }
         }
       };
       this.entityDataListeners.push(listener);
@@ -901,7 +931,11 @@ export class WidgetSubscription implements IWidgetSubscription {
       }
       const forceUpdate = !this.datasources.length;
       this.entityDataListeners.forEach((listener) => {
-        listener.subscriptionTimewindow = this.subscriptionTimewindow;
+        if (this.comparisonEnabled && listener.configDatasource.isAdditional) {
+          listener.subscriptionTimewindow = this.timewindowForComparison;
+        } else {
+          listener.subscriptionTimewindow = this.subscriptionTimewindow;
+        }
         this.ctx.entityDataService.startSubscription(listener);
       });
       if (forceUpdate) {
@@ -1088,9 +1122,50 @@ export class WidgetSubscription implements IWidgetSubscription {
         }
       }
     }
+    if (subscriptionsChanged && this.hasDataPageLink) {
+      subscriptionsChanged = false;
+      this.updateDataSubscriptions();
+    }
     return subscriptionsChanged;
   }
 
+  private updateDataSubscriptions() {
+    this.configuredDatasources = this.ctx.utils.validateDatasources(this.options.datasources);
+    if (!this.ctx.aliasController) {
+      this.hasResolvedData = true;
+      this.prepareDataSubscriptions().subscribe(
+        () => {
+          this.updatePaginatedDataSubscriptions();
+        }
+      );
+    } else {
+      this.ctx.aliasController.resolveDatasources(this.configuredDatasources, this.singleEntity).subscribe(
+        (datasources) => {
+          this.configuredDatasources = datasources;
+          this.prepareDataSubscriptions().subscribe(
+            () => {
+              this.updatePaginatedDataSubscriptions();
+            }
+          );
+        },
+        (err) => {
+          this.notifyDataLoaded();
+        }
+      );
+    }
+  }
+
+  private updatePaginatedDataSubscriptions() {
+    for (let datasourceIndex = 0; datasourceIndex < this.entityDataListeners.length; datasourceIndex++) {
+      const entityDataListener = this.entityDataListeners[datasourceIndex];
+      if (entityDataListener) {
+        const pageLink = entityDataListener.subscription.entityDataSubscriptionOptions.pageLink;
+        const keyFilters = entityDataListener.subscription.entityDataSubscriptionOptions.keyFilters;
+        this.subscribeForPaginatedData(datasourceIndex, pageLink, keyFilters);
+      }
+    }
+  }
+
   isDataResolved(): boolean {
     return this.hasResolvedData;
   }
@@ -1126,7 +1201,7 @@ export class WidgetSubscription implements IWidgetSubscription {
     }
   }
 
-  private updateRealtimeSubscription(subscriptionTimewindow?: SubscriptionTimewindow) {
+  private updateRealtimeSubscription(subscriptionTimewindow?: SubscriptionTimewindow): SubscriptionTimewindow {
     if (subscriptionTimewindow) {
       this.subscriptionTimewindow = subscriptionTimewindow;
     } else {
@@ -1149,11 +1224,15 @@ export class WidgetSubscription implements IWidgetSubscription {
     }
   }
 
-  private updateSubscriptionForComparison() {
-    if (!this.subscriptionTimewindow) {
-      this.subscriptionTimewindow = this.updateRealtimeSubscription();
+  private updateSubscriptionForComparison(subscriptionTimewindow?: SubscriptionTimewindow): SubscriptionTimewindow {
+    if (subscriptionTimewindow) {
+      this.timewindowForComparison = subscriptionTimewindow;
+    } else {
+      if (!this.subscriptionTimewindow) {
+        this.subscriptionTimewindow = this.updateRealtimeSubscription();
+      }
+      this.timewindowForComparison = createTimewindowForComparison(this.subscriptionTimewindow, this.timeForComparison);
     }
-    this.timewindowForComparison = createTimewindowForComparison(this.subscriptionTimewindow, this.timeForComparison);
     this.updateComparisonTimewindow();
     return this.timewindowForComparison;
   }
@@ -1238,13 +1317,29 @@ export class WidgetSubscription implements IWidgetSubscription {
     let index = 0;
     this.datasources.forEach((datasource) => {
       datasource.dataKeys.forEach((dataKey) => {
-        if (datasource.generated) {
+        if (datasource.generated || datasource.isAdditional) {
           dataKey._hash = Math.random();
           dataKey.color = this.ctx.utils.getMaterialColor(index);
         }
         index++;
       });
     });
+    if (this.comparisonEnabled) {
+      this.datasourcePages.forEach(datasourcePage => {
+        datasourcePage.data.forEach((datasource, dIndex) => {
+          if (datasource.isAdditional) {
+            const origDatasource = this.datasourcePages[datasource.origDatasourceIndex].data[dIndex];
+            datasource.dataKeys.forEach((dataKey) => {
+              if (dataKey.settings.comparisonSettings.color) {
+                dataKey.color = dataKey.settings.comparisonSettings.color;
+              }
+              const origDataKey = origDatasource.dataKeys[dataKey.origDataKeyIndex];
+              origDataKey.settings.comparisonSettings.color = dataKey.color;
+            });
+          }
+        });
+      });
+    }
     if (this.displayLegend) {
       this.legendData.keys = this.legendData.keys.sort((key1, key2) => key1.dataKey.label.localeCompare(key2.dataKey.label));
     }
@@ -1260,8 +1355,17 @@ export class WidgetSubscription implements IWidgetSubscription {
     return datasource.dataKeys.map((dataKey, keyIndex) => {
       dataKey.hidden = dataKey.settings.hideDataByDefault ? true : false;
       dataKey.inLegend = dataKey.settings.removeFromLegend ? false : true;
-      dataKey.pattern = dataKey.label;
-      dataKey.label = createLabelFromDatasource(datasource, dataKey.pattern);
+      if (this.comparisonEnabled && dataKey.isAdditional) {
+        if (dataKey.settings.comparisonSettings.comparisonValuesLabel) {
+          dataKey.label = createLabelFromDatasource(datasource, dataKey.settings.comparisonSettings.comparisonValuesLabel);
+        } else {
+          dataKey.label = dataKey.label + ' ' + this.ctx.translate.instant('legend.comparison-time-ago.' + this.timeForComparison);
+        }
+        dataKey.pattern = dataKey.label;
+      } else {
+        dataKey.pattern = dataKey.label;
+        dataKey.label = createLabelFromDatasource(datasource, dataKey.pattern);
+      }
       const datasourceData: DatasourceData = {
         datasource,
         dataKey,

ui-ngx/src/app/core/services/utils.service.ts

@@ -325,7 +325,7 @@ export class UtilsService {
     return dataKey;
   }
 
-  public createAdditionalDataKey(dataKey: DataKey, datasource: Datasource, timeUnit: string,
+  /* public createAdditionalDataKey(dataKey: DataKey, datasource: Datasource, timeUnit: string,
     datasources: Datasource[], additionalKeysNumber: number): DataKey {
     const additionalDataKey = deepClone(dataKey);
     if (dataKey.settings.comparisonSettings.comparisonValuesLabel) {
@@ -343,7 +343,7 @@ export class UtilsService {
     }
     additionalDataKey._hash = Math.random();
     return additionalDataKey;
-  }
+  }*/
 
   public createLabelFromDatasource(datasource: Datasource, pattern: string): string {
     return createLabelFromDatasource(datasource, pattern);

ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts

@@ -156,7 +156,8 @@ export class EntitiesTableWidgetComponent extends PageComponent implements OnIni
     this.pageLink = {
       page: 0,
       pageSize: this.defaultPageSize,
-      textSearch: null
+      textSearch: null,
+      dynamic: true
     };
       // new PageLink(this.defaultPageSize, 0, null, sortOrder);
   }

ui-ngx/src/app/modules/home/components/widget/widget.component.ts

@@ -92,6 +92,7 @@ import { EntityService } from '@core/http/entity.service';
 import { ServicesMap } from '@home/models/services.map';
 import { ResizeObserver } from '@juggle/resize-observer';
 import { EntityDataService } from '@core/api/entity-data.service';
+import { TranslateService } from '@ngx-translate/core';
 
 @Component({
   selector: 'tb-widget',
@@ -165,6 +166,7 @@ export class WidgetComponent extends PageComponent implements OnInit, AfterViewI
               private dashboardService: DashboardService,
               // private datasourceService: DatasourceService,
               private entityDataService: EntityDataService,
+              private translate: TranslateService,
               private utils: UtilsService,
               private raf: RafService,
               private ngZone: NgZone,
@@ -295,6 +297,7 @@ export class WidgetComponent extends PageComponent implements OnInit, AfterViewI
     this.subscriptionContext.timeService = this.timeService;
     this.subscriptionContext.deviceService = this.deviceService;
     this.subscriptionContext.alarmService = this.alarmService;
+    this.subscriptionContext.translate = this.translate;
     // this.subscriptionContext.datasourceService = this.datasourceService;
     this.subscriptionContext.entityDataService = this.entityDataService;
     this.subscriptionContext.utils = this.utils;

ui-ngx/src/app/shared/models/query/query.models.ts

@@ -138,6 +138,7 @@ export interface EntityDataPageLink {
   page: number;
   textSearch?: string;
   sortOrder?: EntityDataSortOrder;
+  dynamic?: boolean;
 }
 
 export function entityDataPageLinkSortDirection(pageLink: EntityDataPageLink): SortDirection {

ui-ngx/src/app/shared/models/telemetry/telemetry.models.ts

@@ -136,6 +136,7 @@ export interface EntityHistoryCmd {
   interval: number;
   limit: number;
   agg: AggregationType;
+  fetchLatestPreviousPoint?: boolean;
 }
 
 export interface LatestValueCmd {
@@ -149,6 +150,7 @@ export interface TimeSeriesCmd {
   interval: number;
   limit: number;
   agg: AggregationType;
+  fetchLatestPreviousPoint?: boolean;
 }
 
 export class EntityDataCmd implements WebsocketCmd {

ui-ngx/src/app/shared/models/widget.models.ts

@@ -235,6 +235,8 @@ export interface DataKey extends KeyInfo {
   usePostProcessing?: boolean;
   hidden?: boolean;
   inLegend?: boolean;
+  isAdditional?: boolean;
+  origDataKeyIndex?: number;
   _hash?: number;
 }
 
@@ -266,6 +268,7 @@ export interface Datasource {
   entityDescription?: string;
   generated?: boolean;
   isAdditional?: boolean;
+  origDatasourceIndex?: number;
   pageLink?: EntityDataPageLink;
   keyFilters?: Array<KeyFilter>;
   entityFilter?: EntityFilter;