From 9f178105cf40ce4a60afad3d9679fd2de2466439 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 8 Sep 2022 18:51:13 +0300 Subject: [PATCH 01/18] UI: Add aggregation configuration to TimeSeries dataKey for latest widget --- .../app/core/api/entity-data-subscription.ts | 33 +++++++++++-- .../src/app/core/api/entity-data.service.ts | 14 +++--- .../src/app/core/api/widget-subscription.ts | 49 +++++++++++++------ .../data-key-config-dialog.component.html | 1 + .../data-key-config-dialog.component.ts | 3 +- .../widget/data-key-config.component.html | 9 ++++ .../widget/data-key-config.component.ts | 25 +++++++++- .../widget/data-keys.component.html | 7 +-- .../widget/data-keys.component.scss | 7 +++ .../components/widget/data-keys.component.ts | 36 +++++++++++++- .../widget/widget-config.component.html | 2 +- .../widget/widget-config.component.ts | 30 ++++++++---- .../home/models/dashboard-component.models.ts | 21 ++++++-- .../models/telemetry/telemetry.models.ts | 19 +++++++ ui-ngx/src/app/shared/models/widget.models.ts | 29 ++++++++++- .../assets/locale/locale.constant-en_US.json | 9 +++- 16 files changed, 246 insertions(+), 48 deletions(-) diff --git a/ui-ngx/src/app/core/api/entity-data-subscription.ts b/ui-ngx/src/app/core/api/entity-data-subscription.ts index 9cffe57ae7..bdcb5b56e0 100644 --- a/ui-ngx/src/app/core/api/entity-data-subscription.ts +++ b/ui-ngx/src/app/core/api/entity-data-subscription.ts @@ -28,6 +28,7 @@ import { TsValue } from '@shared/models/query/query.models'; import { + AggKey, DataKeyType, EntityCountCmd, EntityDataCmd, @@ -55,6 +56,7 @@ declare type DataUpdatedCb = (data: DataSetHolder, dataIndex: number, export interface SubscriptionDataKey { name: string; type: DataKeyType; + aggregationType?: AggregationType; funcBody: string; func?: DataKeyFunction; postFuncBody: string; @@ -95,6 +97,7 @@ export class EntityDataSubscription { private attrFields: Array; private tsFields: Array; private latestValues: Array; + private aggTsValues: Array; private entityDataResolveSubject: Subject; private pageData: PageData; @@ -142,7 +145,8 @@ export class EntityDataSubscription { if (this.datasourceType === DatasourceType.function) { key = `${dataKey.name}_${dataKey.index}_${dataKey.type}${dataKey.latest ? '_latest' : ''}`; } else { - key = `${dataKey.name}_${dataKey.type}${dataKey.latest ? '_latest' : ''}`; + const aggSuffix = dataKey.aggregationType && dataKey.aggregationType !== AggregationType.NONE ? `_${dataKey.aggregationType.toLowerCase()}` : ''; + key = `${dataKey.name}_${dataKey.type}${aggSuffix}${dataKey.latest ? '_latest' : ''}`; } let dataKeysList = this.dataKeys[key] as Array; if (!dataKeysList) { @@ -224,13 +228,15 @@ export class EntityDataSubscription { ); this.tsFields = this.entityDataSubscriptionOptions.dataKeys. - filter(dataKey => dataKey.type === DataKeyType.timeseries && !dataKey.latest).map( + filter(dataKey => dataKey.type === DataKeyType.timeseries && + (!dataKey.aggregationType || dataKey.aggregationType === AggregationType.NONE) && !dataKey.latest).map( dataKey => ({ type: EntityKeyType.TIME_SERIES, key: dataKey.name }) ); if (this.entityDataSubscriptionOptions.type === widgetType.timeseries) { const latestTsFields = this.entityDataSubscriptionOptions.dataKeys. - filter(dataKey => dataKey.type === DataKeyType.timeseries && dataKey.latest).map( + filter(dataKey => dataKey.type === DataKeyType.timeseries && dataKey.latest && + (!dataKey.aggregationType || dataKey.aggregationType === AggregationType.NONE)).map( dataKey => ({ type: EntityKeyType.TIME_SERIES, key: dataKey.name }) ); this.latestValues = this.attrFields.concat(latestTsFields); @@ -238,6 +244,12 @@ export class EntityDataSubscription { this.latestValues = this.attrFields.concat(this.tsFields); } + this.aggTsValues = this.entityDataSubscriptionOptions.dataKeys. + filter(dataKey => dataKey.type === DataKeyType.timeseries && + dataKey.aggregationType && dataKey.aggregationType !== AggregationType.NONE).map( + dataKey => ({ key: dataKey.name, agg: dataKey.aggregationType }) + ); + this.subscriber = new TelemetrySubscriber(this.telemetryService); this.dataCommand = new EntityDataCmd(); @@ -498,6 +510,21 @@ export class EntityDataSubscription { }; } } + if (this.aggTsValues.length > 0) { + if (this.history) { + cmd.aggHistoryCmd = { + keys: this.aggTsValues, + startTs: this.subsTw.fixedWindow.startTimeMs, + endTs: this.subsTw.fixedWindow.endTimeMs + }; + } else { + cmd.aggTsCmd = { + keys: this.aggTsValues, + startTs: this.subsTw.startTs, + timeWindow: this.subsTw.aggregation.timeWindow + }; + } + } } private startFunction() { diff --git a/ui-ngx/src/app/core/api/entity-data.service.ts b/ui-ngx/src/app/core/api/entity-data.service.ts index 3d3112f007..4557d96b0f 100644 --- a/ui-ngx/src/app/core/api/entity-data.service.ts +++ b/ui-ngx/src/app/core/api/entity-data.service.ts @@ -31,6 +31,7 @@ import { Observable, of } from 'rxjs'; export interface EntityDataListener { subscriptionType: widgetType; + useTimewindow?: boolean; subscriptionTimewindow?: SubscriptionTimewindow; latestTsOffset?: number; configDatasource: Datasource; @@ -93,10 +94,10 @@ export class EntityDataService { public startSubscription(listener: EntityDataListener) { if (listener.subscription) { - if (listener.subscriptionType === widgetType.timeseries) { + if (listener.useTimewindow) { listener.subscriptionOptions.subscriptionTimewindow = deepClone(listener.subscriptionTimewindow); - listener.subscriptionOptions.latestTsOffset = listener.latestTsOffset; - } else if (listener.subscriptionType === widgetType.latest) { + } + if (listener.subscriptionType === widgetType.timeseries || listener.subscriptionType === widgetType.latest) { listener.subscriptionOptions.latestTsOffset = listener.latestTsOffset; } listener.subscription.start(); @@ -122,10 +123,10 @@ export class EntityDataService { return of(null); } listener.subscription = new EntityDataSubscription(listener, this.telemetryService, this.utils); - if (listener.subscriptionType === widgetType.timeseries) { + if (listener.useTimewindow) { listener.subscriptionOptions.subscriptionTimewindow = deepClone(listener.subscriptionTimewindow); - listener.subscriptionOptions.latestTsOffset = listener.latestTsOffset; - } else if (listener.subscriptionType === widgetType.latest) { + } + if (listener.subscriptionType === widgetType.timeseries || listener.subscriptionType === widgetType.latest) { listener.subscriptionOptions.latestTsOffset = listener.latestTsOffset; } return listener.subscription.subscribe(); @@ -176,6 +177,7 @@ export class EntityDataService { return { name: dataKey.name, type: dataKey.type, + aggregationType: dataKey.aggregationType, funcBody: dataKey.funcBody, postFuncBody: dataKey.postFuncBody, latest diff --git a/ui-ngx/src/app/core/api/widget-subscription.ts b/ui-ngx/src/app/core/api/widget-subscription.ts index 41aeb5dd14..a01e731b8d 100644 --- a/ui-ngx/src/app/core/api/widget-subscription.ts +++ b/ui-ngx/src/app/core/api/widget-subscription.ts @@ -28,6 +28,7 @@ import { DataSetHolder, Datasource, DatasourceData, + datasourcesHasAggregation, DatasourceType, LegendConfig, LegendData, @@ -95,6 +96,7 @@ export class WidgetSubscription implements IWidgetSubscription { timezone: string; subscriptionTimewindow: SubscriptionTimewindow; useDashboardTimewindow: boolean; + useTimewindow: boolean; tsOffset = 0; hasDataPageLink: boolean; @@ -200,6 +202,7 @@ export class WidgetSubscription implements IWidgetSubscription { this.originalTimewindow = null; this.timeWindow = {}; this.useDashboardTimewindow = options.useDashboardTimewindow; + this.useTimewindow = true; if (this.useDashboardTimewindow) { this.timeWindowConfig = deepClone(options.dashboardTimewindow); } else { @@ -245,15 +248,16 @@ export class WidgetSubscription implements IWidgetSubscription { this.timeWindow = {}; this.useDashboardTimewindow = options.useDashboardTimewindow; this.stateData = options.stateData; - if (this.type === widgetType.latest) { - this.timezone = options.dashboardTimewindow.timezone; - this.updateTsOffset(); - } + this.useTimewindow = this.type === widgetType.timeseries || datasourcesHasAggregation(this.configuredDatasources); if (this.useDashboardTimewindow) { this.timeWindowConfig = deepClone(options.dashboardTimewindow); } else { this.timeWindowConfig = deepClone(options.timeWindowConfig); } + if (this.type === widgetType.latest) { + this.timezone = this.useTimewindow ? this.timeWindowConfig.timezone : options.dashboardTimewindow.timezone; + this.updateTsOffset(); + } this.subscriptionTimewindow = null; this.comparisonEnabled = options.comparisonEnabled && isHistoryTypeTimewindow(this.timeWindowConfig); @@ -443,6 +447,7 @@ export class WidgetSubscription implements IWidgetSubscription { const resolveResultObservables = this.configuredDatasources.map((datasource, index) => { const listener: EntityDataListener = { subscriptionType: this.type, + useTimewindow: this.useTimewindow, configDatasource: datasource, configDatasourceIndex: index, dataLoaded: (pageData, data1, datasourceIndex, pageLink) => { @@ -626,22 +631,31 @@ export class WidgetSubscription implements IWidgetSubscription { } onDashboardTimewindowChanged(newDashboardTimewindow: Timewindow) { - if (this.type === widgetType.timeseries || this.type === widgetType.alarm) { + let doUpdate = false; + let isTimewindowTypeChanged = false; + if (this.useTimewindow) { if (this.useDashboardTimewindow) { + if (this.type === widgetType.latest) { + if (newDashboardTimewindow && this.timezone !== newDashboardTimewindow.timezone) { + this.timezone = newDashboardTimewindow.timezone; + doUpdate = this.updateTsOffset(); + } + } if (!isEqual(this.timeWindowConfig, newDashboardTimewindow) && newDashboardTimewindow) { - const isTimewindowTypeChanged = timewindowTypeChanged(this.timeWindowConfig, newDashboardTimewindow); + isTimewindowTypeChanged = timewindowTypeChanged(this.timeWindowConfig, newDashboardTimewindow); this.timeWindowConfig = deepClone(newDashboardTimewindow); - this.update(isTimewindowTypeChanged); + doUpdate = true; } } } else if (this.type === widgetType.latest) { if (newDashboardTimewindow && this.timezone !== newDashboardTimewindow.timezone) { this.timezone = newDashboardTimewindow.timezone; - if (this.updateTsOffset()) { - this.update(); - } + doUpdate = this.updateTsOffset(); } } + if (doUpdate) { + this.update(isTimewindowTypeChanged); + } } updateDataVisibility(index: number): void { @@ -660,6 +674,12 @@ export class WidgetSubscription implements IWidgetSubscription { updateTimewindowConfig(newTimewindow: Timewindow): void { if (!this.useDashboardTimewindow) { + if (this.type === widgetType.latest) { + if (newTimewindow && this.timezone !== newTimewindow.timezone) { + this.timezone = newTimewindow.timezone; + this.updateTsOffset(); + } + } const isTimewindowTypeChanged = timewindowTypeChanged(this.timeWindowConfig, newTimewindow); this.timeWindowConfig = newTimewindow; this.update(isTimewindowTypeChanged); @@ -874,11 +894,12 @@ export class WidgetSubscription implements IWidgetSubscription { } const datasource = this.configuredDatasources[datasourceIndex]; if (datasource) { - if (this.type === widgetType.timeseries && this.timeWindowConfig) { + if (this.useTimewindow && this.timeWindowConfig) { this.updateRealtimeSubscription(); } entityDataListener = { subscriptionType: this.type, + useTimewindow: this.useTimewindow, configDatasource: datasource, configDatasourceIndex: datasourceIndex, subscriptionTimewindow: this.subscriptionTimewindow, @@ -940,7 +961,7 @@ export class WidgetSubscription implements IWidgetSubscription { private updateDataTimewindow() { if (!this.hasDataPageLink) { - if (this.type === widgetType.timeseries && this.timeWindowConfig) { + if (this.useTimewindow && this.timeWindowConfig) { this.updateRealtimeSubscription(); if (this.comparisonEnabled) { this.updateSubscriptionForComparison(); @@ -952,11 +973,11 @@ export class WidgetSubscription implements IWidgetSubscription { private dataSubscribe() { this.updateDataTimewindow(); if (!this.hasDataPageLink) { - if (this.type === widgetType.timeseries && this.timeWindowConfig && this.subscriptionTimewindow.fixedWindow) { + if (this.useTimewindow && this.timeWindowConfig && this.subscriptionTimewindow.fixedWindow) { this.onDataUpdated(); } const forceUpdate = !this.datasources.length; - const notifyDataLoaded = !this.entityDataListeners.filter((listener) => listener.subscription ? true : false).length; + const notifyDataLoaded = !this.entityDataListeners.filter((listener) => !!listener.subscription).length; this.entityDataListeners.forEach((listener) => { if (this.comparisonEnabled && listener.configDatasource.isAdditional) { listener.subscriptionTimewindow = this.timewindowForComparison; diff --git a/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.html b/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.html index 4f85de08b0..2327d708da 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.html @@ -36,6 +36,7 @@ [dashboard]="data.dashboard" [aliasController]="data.aliasController" [widget]="data.widget" + [widgetType]="data.widgetType" [showPostProcessing]="data.showPostProcessing" [callbacks]="data.callbacks" formControlName="dataKey"> diff --git a/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.ts b/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.ts index 5c67242958..a3d5d8fd65 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.ts @@ -22,7 +22,7 @@ import { AppState } from '@core/core.state'; import { FormBuilder, FormControl, FormGroup, FormGroupDirective, NgForm, Validators } from '@angular/forms'; import { Router } from '@angular/router'; import { DialogComponent } from '@shared/components/dialog.component'; -import { DataKey, Widget } from '@shared/models/widget.models'; +import { DataKey, Widget, widgetType } from '@shared/models/widget.models'; import { DataKeysCallbacks } from './data-keys.component.models'; import { DataKeyConfigComponent } from '@home/components/widget/data-key-config.component'; import { Dashboard } from '@shared/models/dashboard.models'; @@ -35,6 +35,7 @@ export interface DataKeyConfigDialogData { dashboard: Dashboard; aliasController: IAliasController; widget: Widget; + widgetType: widgetType; entityAliasId?: string; showPostProcessing?: boolean; callbacks?: DataKeysCallbacks; diff --git a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html index 19cc902098..5fdbdd2faf 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html @@ -62,6 +62,15 @@ + + datakey.aggregation-type + + + {{ aggregationTypesTranslations.get(aggregationTypes[aggregation]) | translate }} + + + {{ dataKeyFormGroup.get('aggregationType').value ? (dataKeyAggregationTypeHintTranslations.get(aggregationTypes[dataKeyFormGroup.get('aggregationType').value]) | translate) : '' }} +
datakey.data-generation-func
diff --git a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.ts index b97f17f0e0..c737090999 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.ts @@ -18,7 +18,12 @@ import { Component, ElementRef, forwardRef, Input, OnInit, ViewChild } from '@an import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { DataKey, Widget } from '@shared/models/widget.models'; +import { + DataKey, + dataKeyAggregationTypeHintTranslationMap, + Widget, + widgetType +} from '@shared/models/widget.models'; import { ControlValueAccessor, FormBuilder, @@ -43,6 +48,7 @@ import { JsonFormComponentData } from '@shared/components/json-form/json-form-co import { WidgetService } from '@core/http/widget.service'; import { Dashboard } from '@shared/models/dashboard.models'; import { IAliasController } from '@core/api/widget-api.models'; +import { aggregationTranslations, AggregationType } from '@shared/models/time/time.models'; @Component({ selector: 'tb-data-key-config', @@ -65,6 +71,16 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con dataKeyTypes = DataKeyType; + widgetTypes = widgetType; + + aggregations = [AggregationType.NONE, ...Object.keys(AggregationType).filter(type => type !== AggregationType.NONE)]; + + aggregationTypes = AggregationType; + + aggregationTypesTranslations = aggregationTranslations; + + dataKeyAggregationTypeHintTranslations = dataKeyAggregationTypeHintTranslationMap; + @Input() entityAliasId: string; @@ -80,6 +96,9 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con @Input() widget: Widget; + @Input() + widgetType: widgetType; + @Input() dataKeySettingsSchema: any; @@ -155,6 +174,7 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con } this.dataKeyFormGroup = this.fb.group({ name: [null, []], + aggregationType: [null, []], label: [null, [Validators.required]], color: [null, [Validators.required]], units: [null, []], @@ -199,6 +219,9 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con if (this.modelValue.postFuncBody && this.modelValue.postFuncBody.length) { this.modelValue.usePostProcessing = true; } + if (this.widgetType === widgetType.latest && this.modelValue.type === DataKeyType.timeseries && !this.modelValue.aggregationType) { + this.modelValue.aggregationType = AggregationType.NONE; + } this.dataKeyFormGroup.patchValue(this.modelValue, {emitEvent: false}); this.dataKeyFormGroup.get('name').setValidators(this.modelValue.type !== DataKeyType.function && this.modelValue.type !== DataKeyType.count diff --git a/ui-ngx/src/app/modules/home/components/widget/data-keys.component.html b/ui-ngx/src/app/modules/home/components/widget/data-keys.component.html index 3527a5ca2e..5df377cf07 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-keys.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/data-keys.component.html @@ -58,12 +58,7 @@ {{key.label}}
:
-
- f({{key.name}}) - - {{key.name}} - -
+
diff --git a/ui-ngx/src/app/modules/home/components/widget/widget-container.component.html b/ui-ngx/src/app/modules/home/components/widget/widget-container.component.html index a6520e02bd..9f99ffbf29 100644 --- a/ui-ngx/src/app/modules/home/components/widget/widget-container.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/widget-container.component.html @@ -46,6 +46,7 @@
- + -
- timewindow.last - +
+
+ + +
+
+ timewindow.last + +
-
- timewindow.interval - +
+
+ + +
+
+ timewindow.interval + +
+ +
diff --git a/ui-ngx/src/app/shared/components/time/timewindow-panel.component.ts b/ui-ngx/src/app/shared/components/time/timewindow-panel.component.ts index 09fa20aaa3..67ee2ac72b 100644 --- a/ui-ngx/src/app/shared/components/time/timewindow-panel.component.ts +++ b/ui-ngx/src/app/shared/components/time/timewindow-panel.component.ts @@ -36,6 +36,7 @@ export const TIMEWINDOW_PANEL_DATA = new InjectionToken('TimewindowPanelDat export interface TimewindowPanelData { historyOnly: boolean; + quickIntervalOnly: boolean; timewindow: Timewindow; aggregation: boolean; timezone: boolean; @@ -51,6 +52,8 @@ export class TimewindowPanelComponent extends PageComponent implements OnInit { historyOnly = false; + quickIntervalOnly = false; + aggregation = false; timezone = false; @@ -83,6 +86,7 @@ export class TimewindowPanelComponent extends PageComponent implements OnInit { public viewContainerRef: ViewContainerRef) { super(store); this.historyOnly = data.historyOnly; + this.quickIntervalOnly = data.quickIntervalOnly; this.timewindow = data.timewindow; this.aggregation = data.aggregation; this.timezone = data.timezone; @@ -91,6 +95,8 @@ export class TimewindowPanelComponent extends PageComponent implements OnInit { ngOnInit(): void { const hideInterval = this.timewindow.hideInterval || false; + const hideLastInterval = this.timewindow.hideLastInterval || false; + const hideQuickInterval = this.timewindow.hideQuickInterval || false; const hideAggregation = this.timewindow.hideAggregation || false; const hideAggInterval = this.timewindow.hideAggInterval || false; const hideTimezone = this.timewindow.hideTimezone || false; @@ -103,10 +109,11 @@ export class TimewindowPanelComponent extends PageComponent implements OnInit { ? this.timewindow.realtime.realtimeType : RealtimeWindowType.LAST_INTERVAL, disabled: hideInterval }), - timewindowMs: [ - this.timewindow.realtime && typeof this.timewindow.realtime.timewindowMs !== 'undefined' - ? this.timewindow.realtime.timewindowMs : null - ], + timewindowMs: this.fb.control({ + value: this.timewindow.realtime && typeof this.timewindow.realtime.timewindowMs !== 'undefined' + ? this.timewindow.realtime.timewindowMs : null, + disabled: hideInterval || hideLastInterval + }), interval: [ this.timewindow.realtime && typeof this.timewindow.realtime.interval !== 'undefined' ? this.timewindow.realtime.interval : null @@ -114,7 +121,7 @@ export class TimewindowPanelComponent extends PageComponent implements OnInit { quickInterval: this.fb.control({ value: this.timewindow.realtime && typeof this.timewindow.realtime.quickInterval !== 'undefined' ? this.timewindow.realtime.quickInterval : null, - disabled: hideInterval + disabled: hideInterval || hideQuickInterval }) } ), @@ -289,8 +296,40 @@ export class TimewindowPanelComponent extends PageComponent implements OnInit { this.timewindowForm.get('history.fixedTimewindow').enable({emitEvent: false}); this.timewindowForm.get('history.quickInterval').enable({emitEvent: false}); this.timewindowForm.get('realtime.realtimeType').enable({emitEvent: false}); - this.timewindowForm.get('realtime.timewindowMs').enable({emitEvent: false}); - this.timewindowForm.get('realtime.quickInterval').enable({emitEvent: false}); + if (!this.timewindow.hideLastInterval) { + this.timewindowForm.get('realtime.timewindowMs').enable({emitEvent: false}); + } + if (!this.timewindow.hideQuickInterval) { + this.timewindowForm.get('realtime.quickInterval').enable({emitEvent: false}); + } + } + this.timewindowForm.markAsDirty(); + } + + onHideLastIntervalChanged() { + if (this.timewindow.hideLastInterval) { + this.timewindowForm.get('realtime.timewindowMs').disable({emitEvent: false}); + if (!this.timewindow.hideQuickInterval) { + this.timewindowForm.get('realtime.realtimeType').setValue(RealtimeWindowType.INTERVAL); + } + } else { + if (!this.timewindow.hideInterval) { + this.timewindowForm.get('realtime.timewindowMs').enable({emitEvent: false}); + } + } + this.timewindowForm.markAsDirty(); + } + + onHideQuickIntervalChanged() { + if (this.timewindow.hideQuickInterval) { + this.timewindowForm.get('realtime.quickInterval').disable({emitEvent: false}); + if (!this.timewindow.hideLastInterval) { + this.timewindowForm.get('realtime.realtimeType').setValue(RealtimeWindowType.LAST_INTERVAL); + } + } else { + if (!this.timewindow.hideInterval) { + this.timewindowForm.get('realtime.quickInterval').enable({emitEvent: false}); + } } this.timewindowForm.markAsDirty(); } diff --git a/ui-ngx/src/app/shared/components/time/timewindow.component.ts b/ui-ngx/src/app/shared/components/time/timewindow.component.ts index 6f34131241..951f157a59 100644 --- a/ui-ngx/src/app/shared/components/time/timewindow.component.ts +++ b/ui-ngx/src/app/shared/components/time/timewindow.component.ts @@ -82,6 +82,17 @@ export class TimewindowComponent implements OnInit, OnDestroy, ControlValueAcces return this.historyOnlyValue; } + quickIntervalOnlyValue = false; + + @Input() + set quickIntervalOnly(val) { + this.quickIntervalOnlyValue = coerceBooleanProperty(val); + } + + get quickIntervalOnly() { + return this.quickIntervalOnlyValue; + } + aggregationValue = false; @Input() @@ -240,6 +251,7 @@ export class TimewindowComponent implements OnInit, OnDestroy, ControlValueAcces { timewindow: deepClone(this.innerValue), historyOnly: this.historyOnly, + quickIntervalOnly: this.quickIntervalOnly, aggregation: this.aggregation, timezone: this.timezone, isEdit: this.isEdit @@ -278,7 +290,7 @@ export class TimewindowComponent implements OnInit, OnDestroy, ControlValueAcces } writeValue(obj: Timewindow): void { - this.innerValue = initModelFromDefaultTimewindow(obj, this.timeService); + this.innerValue = initModelFromDefaultTimewindow(obj, this.quickIntervalOnly, this.timeService); this.timewindowDisabled = this.isTimewindowDisabled(); this.updateDisplayValue(); } diff --git a/ui-ngx/src/app/shared/models/query/query.models.ts b/ui-ngx/src/app/shared/models/query/query.models.ts index e55beaf7e4..95c756f7f9 100644 --- a/ui-ngx/src/app/shared/models/query/query.models.ts +++ b/ui-ngx/src/app/shared/models/query/query.models.ts @@ -768,7 +768,6 @@ export interface EntityData { latest: {[entityKeyType: string]: {[key: string]: TsValue}}; timeseries: {[key: string]: Array}; aggLatest?: {[aggType: string]: {[key: string]: TsValue}}; - aggFloating?: {[key: string]: Array}; } export interface AlarmData extends AlarmInfo { diff --git a/ui-ngx/src/app/shared/models/telemetry/telemetry.models.ts b/ui-ngx/src/app/shared/models/telemetry/telemetry.models.ts index e519fd4a25..04d98567cf 100644 --- a/ui-ngx/src/app/shared/models/telemetry/telemetry.models.ts +++ b/ui-ngx/src/app/shared/models/telemetry/telemetry.models.ts @@ -188,7 +188,6 @@ export interface AggTimeSeriesCmd { keys: Array; startTs: number; timeWindow: number; - floating: boolean; } export class EntityDataCmd implements WebsocketCmd { @@ -315,6 +314,10 @@ export interface SubscriptionData { [key: string]: [number, any, number?][]; } +export interface AggSubscriptionData { + [aggType: string]: SubscriptionData; +} + export interface SubscriptionDataHolder { data: SubscriptionData; } diff --git a/ui-ngx/src/app/shared/models/time/time.models.ts b/ui-ngx/src/app/shared/models/time/time.models.ts index 2f90d39ca1..d657d5cda4 100644 --- a/ui-ngx/src/app/shared/models/time/time.models.ts +++ b/ui-ngx/src/app/shared/models/time/time.models.ts @@ -96,6 +96,8 @@ export interface Timewindow { displayValue?: string; displayTimezoneAbbr?: string; hideInterval?: boolean; + hideQuickInterval?: boolean; + hideLastInterval?: boolean; hideAggregation?: boolean; hideAggInterval?: boolean; hideTimezone?: boolean; @@ -188,6 +190,8 @@ export function defaultTimewindow(timeService: TimeService): Timewindow { return { displayValue: '', hideInterval: false, + hideLastInterval: false, + hideQuickInterval: false, hideAggregation: false, hideAggInterval: false, hideTimezone: false, @@ -223,10 +227,12 @@ function getTimewindowType(timewindow: Timewindow): TimewindowType { } } -export function initModelFromDefaultTimewindow(value: Timewindow, timeService: TimeService): Timewindow { +export function initModelFromDefaultTimewindow(value: Timewindow, quickIntervalOnly: boolean, timeService: TimeService): Timewindow { const model = defaultTimewindow(timeService); if (value) { model.hideInterval = value.hideInterval; + model.hideLastInterval = value.hideLastInterval; + model.hideQuickInterval = value.hideQuickInterval; model.hideAggregation = value.hideAggregation; model.hideAggInterval = value.hideAggInterval; model.hideTimezone = value.hideTimezone; @@ -281,6 +287,9 @@ export function initModelFromDefaultTimewindow(value: Timewindow, timeService: T } model.timezone = value.timezone; } + if (quickIntervalOnly) { + model.realtime.realtimeType = RealtimeWindowType.INTERVAL; + } return model; } @@ -304,6 +313,8 @@ export function toHistoryTimewindow(timewindow: Timewindow, startTimeMs: number, } return { hideInterval: timewindow.hideInterval || false, + hideLastInterval: timewindow.hideLastInterval || false, + hideQuickInterval: timewindow.hideQuickInterval || false, hideAggregation: timewindow.hideAggregation || false, hideAggInterval: timewindow.hideAggInterval || false, hideTimezone: timewindow.hideTimezone || false, @@ -694,6 +705,8 @@ export function createTimewindowForComparison(subscriptionTimewindow: Subscripti export function cloneSelectedTimewindow(timewindow: Timewindow): Timewindow { const cloned: Timewindow = {}; cloned.hideInterval = timewindow.hideInterval || false; + cloned.hideLastInterval = timewindow.hideLastInterval || false; + cloned.hideQuickInterval = timewindow.hideQuickInterval || false; cloned.hideAggregation = timewindow.hideAggregation || false; cloned.hideAggInterval = timewindow.hideAggInterval || false; cloned.hideTimezone = timewindow.hideTimezone || false; diff --git a/ui-ngx/src/app/shared/models/widget.models.ts b/ui-ngx/src/app/shared/models/widget.models.ts index 3d0b1196a5..9efb47758d 100644 --- a/ui-ngx/src/app/shared/models/widget.models.ts +++ b/ui-ngx/src/app/shared/models/widget.models.ts @@ -338,7 +338,7 @@ export interface Datasource { export function datasourcesHasAggregation(datasources?: Array): boolean { if (datasources) { const foundDatasource = datasources.find(datasource => { - const found = datasource.dataKeys.find(key => key.type === DataKeyType.timeseries && + const found = datasource.dataKeys && datasource.dataKeys.find(key => key.type === DataKeyType.timeseries && key.aggregationType && key.aggregationType !== AggregationType.NONE); return !!found; }); From 6879021027e9f8d97560664c50202cd5813ec77d Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 13 Sep 2022 15:50:50 +0300 Subject: [PATCH 11/18] Fix for multiple commands in same subscription --- ...efaultTbEntityDataSubscriptionService.java | 124 +++++++----------- 1 file changed, 51 insertions(+), 73 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java index 998938b5e2..99f25861a1 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java @@ -29,7 +29,6 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.springframework.web.socket.CloseStatus; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; @@ -70,7 +69,6 @@ import javax.annotation.PreDestroy; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -178,6 +176,8 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc ctx = createSubCtx(session, cmd); } ctx.setCurrentCmd(cmd); + + // Fetch entity list using entity data query if (cmd.getQuery() != null) { if (ctx.getQuery() == null) { log.debug("[{}][{}] Initializing data using query: {}", session.getSessionId(), cmd.getCmdId(), cmd.getQuery()); @@ -209,54 +209,54 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc finalCtx.setRefreshTask(task); } } - if (cmd.getAggHistoryCmd() != null) { - handleAggHistoryCmd(ctx, cmd.getAggHistoryCmd(), cmd.getLatestCmd()); - } else if (cmd.getAggTsCmd() != null) { - handleAggTsCmd(ctx, cmd.getAggTsCmd(), cmd.getLatestCmd()); - } else if (cmd.hasRegularCmds()) { - handleRegularCommands(ctx, cmd); - } else { - checkAndSendInitialData(ctx); - } - } - private void handleRegularCommands(TbEntityDataSubCtx ctx, EntityDataCmd cmd) { - ListenableFuture historyFuture; - if (cmd.getHistoryCmd() != null) { - log.trace("[{}][{}] Going to process history command: {}", ctx.getSessionId(), cmd.getCmdId(), cmd.getHistoryCmd()); - try { - historyFuture = handleHistoryCmd(ctx, cmd.getHistoryCmd()); - } catch (RuntimeException e) { - handleWsCmdRuntimeException(ctx.getSessionId(), e, cmd); - return; + try { + List> cmdFutures = new ArrayList<>(); + if (cmd.getAggHistoryCmd() != null) { + cmdFutures.add(handleAggHistoryCmd(ctx, cmd.getAggHistoryCmd())); } - } else { - historyFuture = Futures.immediateFuture(ctx); - } - Futures.addCallback(historyFuture, new FutureCallback<>() { - @Override - public void onSuccess(@Nullable TbEntityDataSubCtx theCtx) { - try { - if (cmd.getLatestCmd() != null || cmd.getTsCmd() != null) { - if (cmd.getLatestCmd() != null) { - handleLatestCmd(theCtx, cmd.getLatestCmd()); - } - if (cmd.getTsCmd() != null) { - handleTimeSeriesCmd(theCtx, cmd.getTsCmd()); - } - } else { - checkAndSendInitialData(theCtx); + if (cmd.getAggTsCmd() != null) { + cmdFutures.add(handleAggTsCmd(ctx, cmd.getAggTsCmd())); + } + if (cmd.getHistoryCmd() != null) { + cmdFutures.add(handleHistoryCmd(ctx, cmd.getHistoryCmd())); + } + if (cmdFutures.isEmpty()) { + handleRegularCommands(ctx, cmd); + } else { + TbEntityDataSubCtx finalCtx = ctx; + Futures.addCallback(Futures.allAsList(cmdFutures), new FutureCallback<>() { + @Override + public void onSuccess(@Nullable List result) { + handleRegularCommands(finalCtx, cmd); } - } catch (RuntimeException e) { - handleWsCmdRuntimeException(theCtx.getSessionId(), e, cmd); - } + + @Override + public void onFailure(Throwable t) { + log.warn("[{}][{}] Failed to process command", finalCtx.getSessionId(), finalCtx.getCmdId()); + } + }, wsCallBackExecutor); } + } catch (RuntimeException e) { + handleWsCmdRuntimeException(ctx.getSessionId(), e, cmd); + } + } - @Override - public void onFailure(Throwable t) { - log.warn("[{}][{}] Failed to process command", ctx.getSessionId(), cmd.getCmdId()); + private void handleRegularCommands(TbEntityDataSubCtx ctx, EntityDataCmd cmd) { + try { + if (cmd.getLatestCmd() != null || cmd.getTsCmd() != null) { + if (cmd.getLatestCmd() != null) { + handleLatestCmd(ctx, cmd.getLatestCmd()); + } + if (cmd.getTsCmd() != null) { + handleTimeSeriesCmd(ctx, cmd.getTsCmd()); + } + } else { + checkAndSendInitialData(ctx); } - }, wsCallBackExecutor); + } catch (RuntimeException e) { + handleWsCmdRuntimeException(ctx.getSessionId(), e, cmd); + } } private void checkAndSendInitialData(@Nullable TbEntityDataSubCtx theCtx) { @@ -267,30 +267,30 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc } } - private void handleAggHistoryCmd(TbEntityDataSubCtx ctx, AggHistoryCmd cmd, LatestValueCmd latestCmd) { + private ListenableFuture handleAggHistoryCmd(TbEntityDataSubCtx ctx, AggHistoryCmd cmd) { var keys = cmd.getKeys(); long interval = cmd.getEndTs() - cmd.getStartTs(); List queries = keys.stream().map(key -> new BaseReadTsKvQuery( key.getKey(), cmd.getStartTs(), cmd.getEndTs(), interval, 1, key.getAgg() )).distinct().collect(Collectors.toList()); - handleAggCmd(ctx, cmd.getKeys(), queries, cmd.getStartTs(), cmd.getEndTs(), latestCmd, false); + return handleAggCmd(ctx, cmd.getKeys(), queries, cmd.getStartTs(), cmd.getEndTs(), false); } - private void handleAggTsCmd(TbEntityDataSubCtx ctx, AggTimeSeriesCmd cmd, LatestValueCmd latestCmd) { + private ListenableFuture handleAggTsCmd(TbEntityDataSubCtx ctx, AggTimeSeriesCmd cmd) { long endTs = cmd.getStartTs() + cmd.getTimeWindow(); List queries = cmd.getKeys().stream() .map(key -> new BaseReadTsKvQuery(key.getKey(), cmd.getStartTs(), endTs, cmd.getTimeWindow(), 1, key.getAgg())) .distinct().collect(Collectors.toList()); - handleAggCmd(ctx, cmd.getKeys(), queries, cmd.getStartTs(), endTs, latestCmd, true); + return handleAggCmd(ctx, cmd.getKeys(), queries, cmd.getStartTs(), endTs, true); } - private void handleAggCmd(TbEntityDataSubCtx ctx, List keys, List queries, - long startTs, long endTs, LatestValueCmd latestCmd, boolean subscribe) { + private ListenableFuture handleAggCmd(TbEntityDataSubCtx ctx, List keys, List queries, + long startTs, long endTs, boolean subscribe) { Map>> fetchResultMap = new HashMap<>(); List entityDataList = ctx.getData().getData(); entityDataList.forEach(entityData -> fetchResultMap.put(entityData, tsService.findAllByQueries(ctx.getTenantId(), entityData.getEntityId(), queries))); - var mainFuture = Futures.transform(Futures.allAsList(fetchResultMap.values()), f -> { + return Futures.transform(Futures.allAsList(fetchResultMap.values()), f -> { // Map that holds last ts for each key for each entity. Map> lastTsEntityMap = new HashMap<>(); fetchResultMap.forEach((entityData, future) -> { @@ -333,28 +333,6 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc } return ctx; }, wsCallBackExecutor); - - Futures.addCallback(mainFuture, new FutureCallback<>() { - - @Override - public void onSuccess(@Nullable TbEntityDataSubCtx theCtx) { - if (latestCmd != null) { - if (subscribe) { - var commandEntityKeys = new HashSet<>(latestCmd.getKeys()); - var alreadySubscribedKeys = keys.stream().map(key -> new EntityKey(EntityKeyType.TIME_SERIES, key.getKey())).collect(Collectors.toList()); - if (commandEntityKeys.removeAll(alreadySubscribedKeys)) { - latestCmd.setKeys(new ArrayList<>(commandEntityKeys)); - } - } - handleLatestCmd(ctx, latestCmd); - } - } - - @Override - public void onFailure(Throwable t) { - log.warn("[{}][{}] Failed to process command", ctx.getSessionId(), ctx.getCmdId()); - } - }, wsCallBackExecutor); } private void handleWsCmdRuntimeException(String sessionId, RuntimeException e, EntityDataCmd cmd) { From cb307ce9a5b533098924e45d89452412525312c8 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 13 Sep 2022 17:01:52 +0300 Subject: [PATCH 12/18] UI: Add latest data support to tooltip value format function of flot widget. --- .../app/core/api/entity-data-subscription.ts | 13 +++-- .../widget/lib/flot-widget.models.ts | 4 +- .../home/components/widget/lib/flot-widget.ts | 49 ++++++++++++++++--- .../chart/flot-key-settings.component.html | 2 +- .../chart/flot-widget-settings.component.html | 2 +- 5 files changed, 53 insertions(+), 17 deletions(-) diff --git a/ui-ngx/src/app/core/api/entity-data-subscription.ts b/ui-ngx/src/app/core/api/entity-data-subscription.ts index 0d6c534e4c..8032a32973 100644 --- a/ui-ngx/src/app/core/api/entity-data-subscription.ts +++ b/ui-ngx/src/app/core/api/entity-data-subscription.ts @@ -457,7 +457,7 @@ export class EntityDataSubscription { } this.prepareSubscriptionTimewindow(); - this.prepareData(); + this.prepareData(true); if (this.datasourceType === DatasourceType.entity) { this.subsCommand = new EntityDataCmd(); @@ -567,7 +567,7 @@ export class EntityDataSubscription { this.generateData(true); } - private prepareData() { + private prepareData(isUpdate: boolean) { if (this.timeseriesTimer) { clearTimeout(this.timeseriesTimer); this.timeseriesTimer = null; @@ -641,11 +641,14 @@ export class EntityDataSubscription { for (let dataIndex = 0; dataIndex < this.pageData.data.length; dataIndex++) { this.onAggData(aggSubscriptionData, DataKeyType.timeseries, dataIndex, true, this.entityDataSubscriptionOptions.type === widgetType.timeseries, true, - (data1, dataIndex1, dataKeyIndex) => { + (data, dataIndex1, dataKeyIndex, detectChanges, isLatest) => { if (!this.data[dataIndex1]) { this.data[dataIndex1] = []; } - this.data[dataIndex1][dataKeyIndex] = data1; + this.data[dataIndex1][dataKeyIndex] = data; + if (isUpdate) { + this.notifyListener(data, dataIndex1, dataKeyIndex, detectChanges, isLatest); + } }); } } @@ -706,7 +709,7 @@ export class EntityDataSubscription { this.pageData = pageData; if (this.entityDataSubscriptionOptions.isPaginatedDataSubscription) { - this.prepareData(); + this.prepareData(false); } else if (isInitialData) { this.resetData(); } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.models.ts index 595384abaf..78c9245614 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.models.ts @@ -17,7 +17,7 @@ // tslint:disable-next-line:no-reference /// -import { DataKey, Datasource, DatasourceData, JsonSettingsSchema } from '@shared/models/widget.models'; +import { DataKey, Datasource, DatasourceData, FormattedData, JsonSettingsSchema } from '@shared/models/widget.models'; import { DataKeyType } from '@shared/models/telemetry/telemetry.models'; import { ComparisonDuration } from '@shared/models/time/time.models'; @@ -25,7 +25,7 @@ export declare type ChartType = 'line' | 'pie' | 'bar' | 'state' | 'graph'; export declare type TbFlotSettings = TbFlotBaseSettings & TbFlotGraphSettings & TbFlotBarSettings & TbFlotPieSettings; -export declare type TooltipValueFormatFunction = (value: any) => string; +export declare type TooltipValueFormatFunction = (value: any, latestData: FormattedData) => string; export declare type TbFlotTicksFormatterFunction = (t: number, a?: TbFlotPlotAxis) => string; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.ts b/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.ts index bf59e36804..c601df72d4 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.ts @@ -18,7 +18,7 @@ import { WidgetContext } from '@home/models/widget-component.models'; import { createLabelFromDatasource, - deepClone, + deepClone, formattedDataFormDatasourceData, insertVariable, isDefined, isDefinedAndNotNull, @@ -28,7 +28,14 @@ import { isUndefined } from '@app/core/utils'; import { IWidgetSubscription, WidgetSubscriptionOptions } from '@core/api/widget-api.models'; -import { DataKey, Datasource, DatasourceData, DatasourceType, widgetType } from '@app/shared/models/widget.models'; +import { + DataKey, + Datasource, + DatasourceData, + DatasourceType, + FormattedData, + widgetType +} from '@app/shared/models/widget.models'; import { ChartType, TbFlotAxisOptions, @@ -88,6 +95,8 @@ export class TbFlot { private latestDataThresholds: TbFlotThresholdMarking[]; private attributesThresholds: TbFlotThresholdMarking[]; + private latestData: FormattedData[]; + private labelPatternsSourcesSubscription: IWidgetSubscription; private labelPatternsSourcesData: DatasourceData[]; @@ -379,7 +388,7 @@ export class TbFlot { let tooltipValueFormatFunction: TooltipValueFormatFunction = null; if (this.settings.tooltipValueFormatter && this.settings.tooltipValueFormatter.length) { try { - tooltipValueFormatFunction = new Function('value', this.settings.tooltipValueFormatter) as TooltipValueFormatFunction; + tooltipValueFormatFunction = new Function('value', 'latestData', this.settings.tooltipValueFormatter) as TooltipValueFormatFunction; } catch (e) { tooltipValueFormatFunction = null; } @@ -392,7 +401,7 @@ export class TbFlot { series.dataKey.tooltipValueFormatFunction = tooltipValueFormatFunction; if (keySettings.tooltipValueFormatter && keySettings.tooltipValueFormatter.length) { try { - series.dataKey.tooltipValueFormatFunction = new Function('value', + series.dataKey.tooltipValueFormatFunction = new Function('value', 'latestData', keySettings.tooltipValueFormatter) as TooltipValueFormatFunction; } catch (e) { series.dataKey.tooltipValueFormatFunction = tooltipValueFormatFunction; @@ -546,6 +555,13 @@ export class TbFlot { } this.latestDataThresholds = this.thresholdsSourcesDataUpdated(allThresholds, this.subscription.latestData, true); this.options.grid.markings = allThresholds.concat(this.latestDataThresholds); + if (this.subscription.latestData) { + this.latestData = formattedDataFormDatasourceData(this.subscription.latestData); + } else { + this.latestData = []; + } + } else if (this.chartType === 'pie') { + this.latestData = formattedDataFormDatasourceData(this.subscription.data); } this.checkMouseEvents(); @@ -653,6 +669,7 @@ export class TbFlot { this.updateData(); } } else if (this.chartType === 'pie') { + this.latestData = formattedDataFormDatasourceData(this.subscription.data); if (this.animatedPie) { this.nextPieDataAnimation(true); } else { @@ -683,6 +700,11 @@ export class TbFlot { this.plot.getOptions().grid.markings = this.options.grid.markings; this.updateData(); } + if (this.subscription.latestData) { + this.latestData = formattedDataFormDatasourceData(this.subscription.latestData); + } else { + this.latestData = []; + } } } else if (this.isMouseInteraction && this.plot) { this.latestUpdateTimeoutHandle = setTimeout(this.latestDataUpdate.bind(this), 30); @@ -690,6 +712,14 @@ export class TbFlot { } } + private latestDataByDataIndex(index: number): FormattedData { + if (this.latestData[index]) { + return this.latestData[index]; + } else { + return {} as FormattedData; + } + } + private scalingPieRadius() { let scalingLine; this.ctx.width > this.ctx.height ? scalingLine = this.ctx.height : scalingLine = this.ctx.width; @@ -1004,7 +1034,8 @@ export class TbFlot { private seriesInfoDiv(label: string, color: string, value: any, units: string, trackDecimals: number, active: boolean, - percent: number, valueFormatFunction: TooltipValueFormatFunction): JQuery { + percent: number, seriesIndex: number, + valueFormatFunction: TooltipValueFormatFunction): JQuery { const divElement = $('
'); divElement.css({ display: 'flex', @@ -1034,7 +1065,7 @@ export class TbFlot { divElement.append(labelSpan); let valueContent: string; if (valueFormatFunction) { - valueContent = valueFormatFunction(value); + valueContent = valueFormatFunction(value, this.latestDataByDataIndex(seriesIndex)); } else { valueContent = this.ctx.utils.formatValue(value, trackDecimals, units); } @@ -1059,7 +1090,8 @@ export class TbFlot { const units = seriesHoverInfo.units && seriesHoverInfo.units.length ? seriesHoverInfo.units : this.trackUnits; const decimals = isDefinedAndNotNull(seriesHoverInfo.decimals) ? seriesHoverInfo.decimals : this.trackDecimals; const divElement = this.seriesInfoDiv(seriesHoverInfo.label, seriesHoverInfo.color, - seriesHoverInfo.value, units, decimals, seriesHoverInfo.index === seriesIndex, null, seriesHoverInfo.tooltipValueFormatFunction); + seriesHoverInfo.value, units, decimals, seriesHoverInfo.index === seriesIndex, null, seriesHoverInfo.index, + seriesHoverInfo.tooltipValueFormatFunction); return divElement.prop('outerHTML'); } @@ -1086,7 +1118,8 @@ export class TbFlot { const units = item.series.dataKey.units && item.series.dataKey.units.length ? item.series.dataKey.units : this.trackUnits; const decimals = isDefinedAndNotNull(item.series.dataKey.decimals) ? item.series.dataKey.decimals : this.trackDecimals; const divElement = this.seriesInfoDiv(item.series.dataKey.label, item.series.dataKey.color, - item.datapoint[1][0][1], units, decimals, true, item.series.percent, item.series.dataKey.tooltipValueFormatFunction); + item.datapoint[1][0][1], units, decimals, true, item.series.percent, 0, + item.series.dataKey.tooltipValueFormatFunction); return divElement.prop('outerHTML'); } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/flot-key-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/flot-key-settings.component.html index 05879cb124..5741bfae9f 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/flot-key-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/flot-key-settings.component.html @@ -123,7 +123,7 @@ diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/flot-widget-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/flot-widget-settings.component.html index beebf38268..06b9bc5f86 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/flot-widget-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/flot-widget-settings.component.html @@ -96,7 +96,7 @@ From 9d30b7558fbd472003a9b0f840b65fe4f05b7f56 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 14 Sep 2022 14:22:43 +0300 Subject: [PATCH 13/18] Support of Comparison Queries --- ...efaultTbEntityDataSubscriptionService.java | 73 +++++++++++++------ .../subscription/ReadTsKvQueryInfo.java | 29 ++++++++ .../service/telemetry/cmd/v2/AggKey.java | 6 ++ .../common/data/kv/BaseReadTsKvQuery.java | 3 +- .../server/common/data/kv/BaseTsKvQuery.java | 5 ++ .../common/data/kv/ReadTsKvQueryResult.java | 4 +- .../server/common/data/kv/TsKvQuery.java | 2 + .../common/data/query/ComparisonTsValue.java | 29 ++++++++ .../server/common/data/query/EntityData.java | 4 +- ...stractChunkedAggregationTimeseriesDao.java | 2 +- .../sqlts/BaseAbstractSqlTimeseriesDao.java | 2 +- .../timescale/TimescaleTimeseriesDao.java | 2 +- .../dao/timeseries/BaseTimeseriesService.java | 1 + .../CassandraBaseTimeseriesDao.java | 6 +- 14 files changed, 131 insertions(+), 37 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/ReadTsKvQueryInfo.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/query/ComparisonTsValue.java diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java index 99f25861a1..b583446f17 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java @@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmDataQuery; +import org.thingsboard.server.common.data.query.ComparisonTsValue; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityKey; @@ -74,6 +75,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -268,28 +270,36 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc } private ListenableFuture handleAggHistoryCmd(TbEntityDataSubCtx ctx, AggHistoryCmd cmd) { - var keys = cmd.getKeys(); - long interval = cmd.getEndTs() - cmd.getStartTs(); - List queries = keys.stream().map(key -> new BaseReadTsKvQuery( - key.getKey(), cmd.getStartTs(), cmd.getEndTs(), interval, 1, key.getAgg() - )).distinct().collect(Collectors.toList()); + ConcurrentMap queries = new ConcurrentHashMap<>(); + for (AggKey key : cmd.getKeys()) { + if (key.getPreviousValueOnly() == null || !key.getPreviousValueOnly()) { + var query = new BaseReadTsKvQuery(key.getKey(), cmd.getStartTs(), cmd.getEndTs(), cmd.getEndTs() - cmd.getStartTs(), 1, key.getAgg()); + queries.put(query.getId(), new ReadTsKvQueryInfo(key, query, false)); + } + if (key.getPreviousStartTs() != null && key.getPreviousEndTs() != null && key.getPreviousEndTs() >= key.getPreviousStartTs()) { + var query = new BaseReadTsKvQuery(key.getKey(), key.getPreviousStartTs(), key.getPreviousEndTs(), key.getPreviousEndTs() - key.getPreviousStartTs(), 1, key.getAgg()); + queries.put(query.getId(), new ReadTsKvQueryInfo(key, query, true)); + } + } return handleAggCmd(ctx, cmd.getKeys(), queries, cmd.getStartTs(), cmd.getEndTs(), false); } private ListenableFuture handleAggTsCmd(TbEntityDataSubCtx ctx, AggTimeSeriesCmd cmd) { - long endTs = cmd.getStartTs() + cmd.getTimeWindow(); - List queries = cmd.getKeys().stream() - .map(key -> new BaseReadTsKvQuery(key.getKey(), cmd.getStartTs(), endTs, cmd.getTimeWindow(), 1, key.getAgg())) - .distinct().collect(Collectors.toList()); - return handleAggCmd(ctx, cmd.getKeys(), queries, cmd.getStartTs(), endTs, true); + ConcurrentMap queries = new ConcurrentHashMap<>(); + for (AggKey key : cmd.getKeys()) { + var query = new BaseReadTsKvQuery(key.getKey(), cmd.getStartTs(), cmd.getStartTs() + cmd.getTimeWindow(), cmd.getTimeWindow(), 1, key.getAgg()); + queries.put(query.getId(), new ReadTsKvQueryInfo(key, query, false)); + } + return handleAggCmd(ctx, cmd.getKeys(), queries, cmd.getStartTs(), cmd.getStartTs() + cmd.getTimeWindow(), true); } - private ListenableFuture handleAggCmd(TbEntityDataSubCtx ctx, List keys, List queries, + private ListenableFuture handleAggCmd(TbEntityDataSubCtx ctx, List keys, ConcurrentMap queries, long startTs, long endTs, boolean subscribe) { Map>> fetchResultMap = new HashMap<>(); List entityDataList = ctx.getData().getData(); + List queryList = queries.values().stream().map(ReadTsKvQueryInfo::getQuery).collect(Collectors.toList()); entityDataList.forEach(entityData -> fetchResultMap.put(entityData, - tsService.findAllByQueries(ctx.getTenantId(), entityData.getEntityId(), queries))); + tsService.findAllByQueries(ctx.getTenantId(), entityData.getEntityId(), queryList))); return Futures.transform(Futures.allAsList(fetchResultMap.values()), f -> { // Map that holds last ts for each key for each entity. Map> lastTsEntityMap = new HashMap<>(); @@ -301,13 +311,19 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc List queryResults = future.get(); if (queryResults != null) { for (ReadTsKvQueryResult queryResult : queryResults) { - entityData.getAggLatest().computeIfAbsent(queryResult.getAgg(), agg -> new HashMap<>()).put(queryResult.getKey(), queryResult.toTsValue()); - lastTsMap.put(queryResult.getKey(), queryResult.getLastEntryTs()); + ReadTsKvQueryInfo queryInfo = queries.get(queryResult.getQueryId()); + ComparisonTsValue comparisonTsValue = entityData.getAggLatest().computeIfAbsent(queryInfo.getKey().getId(), agg -> new ComparisonTsValue()); + if (queryInfo.isPrevious()) { + comparisonTsValue.setPrevious(queryResult.toTsValue()); + } else { + comparisonTsValue.setCurrent(queryResult.toTsValue()); + lastTsMap.put(queryInfo.getQuery().getKey(), queryResult.getLastEntryTs()); + } } } // Populate with empty values if no data found. keys.forEach(key -> { - entityData.getAggLatest().computeIfAbsent(key.getAgg(), agg -> new HashMap<>()).putIfAbsent(key.getKey(), TsValue.EMPTY); + entityData.getAggLatest().putIfAbsent(key.getId(), new ComparisonTsValue(TsValue.EMPTY, TsValue.EMPTY)); }); } catch (InterruptedException | ExecutionException e) { log.warn("[{}][{}][{}] Failed to fetch historical data", ctx.getSessionId(), ctx.getCmdId(), entityData.getEntityId(), e); @@ -507,16 +523,26 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc } private ListenableFuture handleGetTsCmd(TbEntityDataSubCtx ctx, GetTsCmd cmd, boolean subscribe) { + Map queriesKeys = new ConcurrentHashMap<>(); + List keys = cmd.getKeys(); List finalTsKvQueryList; - List tsKvQueryList = keys.stream().map(key -> new BaseReadTsKvQuery( - key, cmd.getStartTs(), cmd.getEndTs(), cmd.getInterval(), getLimit(cmd.getLimit()), cmd.getAgg() - )).collect(Collectors.toList()); + List tsKvQueryList = keys.stream().map(key -> { + var query = new BaseReadTsKvQuery( + key, cmd.getStartTs(), cmd.getEndTs(), cmd.getInterval(), getLimit(cmd.getLimit()), cmd.getAgg() + ); + queriesKeys.put(query.getId(), query.getKey()); + return query; + }).collect(Collectors.toList()); if (cmd.isFetchLatestPreviousPoint()) { finalTsKvQueryList = new ArrayList<>(tsKvQueryList); - finalTsKvQueryList.addAll(keys.stream().map(key -> new BaseReadTsKvQuery( - key, cmd.getStartTs() - TimeUnit.DAYS.toMillis(365), cmd.getStartTs(), cmd.getInterval(), 1, cmd.getAgg() - )).collect(Collectors.toList())); + finalTsKvQueryList.addAll(keys.stream().map(key -> { + var query = new BaseReadTsKvQuery( + key, cmd.getStartTs() - TimeUnit.DAYS.toMillis(365), cmd.getStartTs(), cmd.getInterval(), 1, cmd.getAgg()); + queriesKeys.put(query.getId(), query.getKey()); + return query; + } + ).collect(Collectors.toList())); } else { finalTsKvQueryList = tsKvQueryList; } @@ -535,8 +561,9 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc List queryResults = future.get(); if (queryResults != null) { for (ReadTsKvQueryResult queryResult : queryResults) { - entityData.getTimeseries().put(queryResult.getKey(), queryResult.toTsValues()); - lastTsMap.put(queryResult.getKey(), queryResult.getLastEntryTs()); + String queryKey = queriesKeys.get(queryResult.getQueryId()); + entityData.getTimeseries().put(queryKey, queryResult.toTsValues()); + lastTsMap.put(queryKey, queryResult.getLastEntryTs()); } } // Populate with empty values if no data found. diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/ReadTsKvQueryInfo.java b/application/src/main/java/org/thingsboard/server/service/subscription/ReadTsKvQueryInfo.java new file mode 100644 index 0000000000..28ca2f6729 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/ReadTsKvQueryInfo.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.Data; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.service.telemetry.cmd.v2.AggKey; + +@Data +public class ReadTsKvQueryInfo { + + private final AggKey key; + private final ReadTsKvQuery query; + private final boolean previous; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/AggKey.java b/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/AggKey.java index aaf3997193..d567149d01 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/AggKey.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/AggKey.java @@ -21,6 +21,12 @@ import org.thingsboard.server.common.data.kv.Aggregation; @Data public class AggKey { + private int id; private String key; private Aggregation agg; + + private Long previousStartTs; + private Long previousEndTs; + private Boolean previousValueOnly; + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java index fc111da8e2..29b98d5e83 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java @@ -31,8 +31,7 @@ public class BaseReadTsKvQuery extends BaseTsKvQuery implements ReadTsKvQuery { this(key, startTs, endTs, interval, limit, aggregation, "DESC"); } - public BaseReadTsKvQuery(String key, long startTs, long endTs, long interval, int limit, Aggregation aggregation, - String order) { + public BaseReadTsKvQuery(String key, long startTs, long endTs, long interval, int limit, Aggregation aggregation, String order) { super(key, startTs, endTs); this.interval = interval; this.limit = limit; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseTsKvQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseTsKvQuery.java index 4102b30a6f..245af53d55 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseTsKvQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseTsKvQuery.java @@ -20,11 +20,16 @@ import lombok.Data; @Data public class BaseTsKvQuery implements TsKvQuery { + private static final ThreadLocal idSeq = ThreadLocal.withInitial(() -> 0); + + private final int id; private final String key; private final long startTs; private final long endTs; public BaseTsKvQuery(String key, long startTs, long endTs) { + this.id = idSeq.get(); + idSeq.set(id + 1); this.key = key; this.startTs = startTs; this.endTs = endTs; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/ReadTsKvQueryResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/ReadTsKvQueryResult.java index 26a8a0c2c5..9c7a2f1754 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/ReadTsKvQueryResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/ReadTsKvQueryResult.java @@ -24,9 +24,7 @@ import java.util.List; @Data public class ReadTsKvQueryResult { - private final String key; - // Holds the aggregation from the query - private final Aggregation agg; + private final int queryId; // Holds the data list; private final List data; // Holds the max ts of the records that match aggregation intervals (not the ts of the aggregation window, but the ts of the last record among all the intervals) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvQuery.java index cf01c322a5..78bf01c0e9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvQuery.java @@ -17,6 +17,8 @@ package org.thingsboard.server.common.data.kv; public interface TsKvQuery { + int getId(); + String getKey(); long getStartTs(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/ComparisonTsValue.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/ComparisonTsValue.java new file mode 100644 index 0000000000..301fbdb0ee --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/ComparisonTsValue.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.query; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class ComparisonTsValue { + + private TsValue current; + private TsValue previous; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java index f1445faaab..2b24866ac4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java @@ -15,11 +15,9 @@ */ package org.thingsboard.server.common.data.query; -import lombok.AllArgsConstructor; import lombok.Data; import lombok.RequiredArgsConstructor; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.kv.Aggregation; import java.util.Map; @@ -30,7 +28,7 @@ public class EntityData { private final EntityId entityId; private final Map> latest; private final Map timeseries; - private final Map> aggLatest; + private final Map aggLatest; public EntityData(EntityId entityId, Map> latest, Map timeseries) { this(entityId, latest, timeseries, null); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index b5905dcf7d..9acbfe85fb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -149,7 +149,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq tsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(query.getKey())); List tsKvEntries = DaoUtil.convertDataList(tsKvEntities); long lastTs = tsKvEntries.stream().map(TsKvEntry::getTs).max(Long::compare).orElse(query.getStartTs()); - return new ReadTsKvQueryResult(query.getKey(), query.getAggregation(), tsKvEntries, lastTs); + return new ReadTsKvQueryResult(query.getId(), tsKvEntries, lastTs); } Optional findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java index 27bb3307be..1946198505 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java @@ -99,7 +99,7 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni if (lastTs.isEmpty()) { lastTs = data.stream().map(AbstractTsKvEntity::getTs).filter(Objects::nonNull).max(Long::compare); } - return new ReadTsKvQueryResult(query.getKey(), query.getAggregation(), DaoUtil.convertDataList(data), lastTs.orElse(query.getStartTs())); + return new ReadTsKvQueryResult(query.getId(), DaoUtil.convertDataList(data), lastTs.orElse(query.getStartTs())); } }, service); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 9f2763fde9..76dabb24f6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -181,7 +181,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements timescaleTsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(strKey)); var tsKvEntries = DaoUtil.convertDataList(timescaleTsKvEntities); long lastTs = tsKvEntries.stream().map(TsKvEntry::getTs).max(Long::compare).orElse(query.getStartTs()); - return new ReadTsKvQueryResult(query.getKey(), query.getAggregation(), tsKvEntries, lastTs); + return new ReadTsKvQueryResult(query.getId(), tsKvEntries, lastTs); } private List> findAllAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long timeBucket, Aggregation aggregation) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index b785ae2f9d..b4cb27cb53 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -46,6 +46,7 @@ import org.thingsboard.server.dao.service.Validator; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.StringUtils.isBlank; diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 01450bbb55..35eb5b8a18 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -285,7 +285,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD @Override public ReadTsKvQueryResult apply(@Nullable List> input) { if (input == null) { - return new ReadTsKvQueryResult(query.getKey(), query.getAggregation(), Collections.emptyList(), query.getStartTs()); + return new ReadTsKvQueryResult(query.getId(), Collections.emptyList(), query.getStartTs()); } else { long maxTs = query.getStartTs(); List data = new ArrayList<>(); @@ -296,7 +296,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD data.add(tsKvEntryAggWrapper.getEntry()); } } - return new ReadTsKvQueryResult(query.getKey(), query.getAggregation(), data, maxTs); + return new ReadTsKvQueryResult(query.getId(), data, maxTs); } } @@ -333,7 +333,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD if (tsKvEntries != null) { lastTs = tsKvEntries.stream().map(TsKvEntry::getTs).max(Long::compare).orElse(query.getStartTs()); } - return new ReadTsKvQueryResult(query.getKey(), query.getAggregation(), tsKvEntries, lastTs); + return new ReadTsKvQueryResult(query.getId(), tsKvEntries, lastTs); }, MoreExecutors.directExecutor()); } From b5383aa42175f410724ab3d64d5bec9cb95f33f5 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 14 Sep 2022 17:49:37 +0300 Subject: [PATCH 14/18] UI: Update timeseries aggregation data structures. Implement timeseries aggregation delta calculation. --- ui-ngx/src/app/core/api/data-aggregator.ts | 169 ++++--- .../app/core/api/entity-data-subscription.ts | 426 +++++++++++------- .../src/app/core/api/entity-data.service.ts | 30 +- .../widget/data-key-config.component.html | 72 ++- .../widget/data-key-config.component.scss | 74 +++ .../widget/data-key-config.component.ts | 108 ++++- .../app/shared/models/query/query.models.ts | 7 +- .../models/telemetry/telemetry.models.ts | 8 +- ui-ngx/src/app/shared/models/widget.models.ts | 20 +- .../assets/locale/locale.constant-en_US.json | 6 +- 10 files changed, 619 insertions(+), 301 deletions(-) diff --git a/ui-ngx/src/app/core/api/data-aggregator.ts b/ui-ngx/src/app/core/api/data-aggregator.ts index 697fa2489a..6e77a0a161 100644 --- a/ui-ngx/src/app/core/api/data-aggregator.ts +++ b/ui-ngx/src/app/core/api/data-aggregator.ts @@ -16,8 +16,7 @@ import { AggKey, - AggSubscriptionData, - SubscriptionData + IndexedSubscriptionData, } from '@app/shared/models/telemetry/telemetry.models'; import { AggregationType, @@ -32,7 +31,7 @@ import { UtilsService } from '@core/services/utils.service'; import { deepClone, isDefinedAndNotNull, isNumber, isNumeric } from '@core/utils'; import Timeout = NodeJS.Timeout; -export declare type onAggregatedData = (data: AggSubscriptionData, detectChanges: boolean) => void; +export declare type onAggregatedData = (data: IndexedSubscriptionData, detectChanges: boolean) => void; interface AggData { count: number; @@ -71,12 +70,12 @@ class AggDataMap { } class AggregationMap { - aggMap: {[aggKey: string]: AggDataMap} = {}; + aggMap: {[id: number]: AggDataMap} = {}; detectRangeChanged(): boolean { let changed = false; - for (const aggKey of Object.keys(this.aggMap)) { - const aggDataMap = this.aggMap[aggKey]; + for (const id of Object.keys(this.aggMap)) { + const aggDataMap = this.aggMap[id]; if (aggDataMap.rangeChanged) { changed = true; aggDataMap.rangeChanged = false; @@ -86,8 +85,8 @@ class AggregationMap { } clearRangeChangedFlags() { - for (const aggKey of Object.keys(this.aggMap)) { - this.aggMap[aggKey].rangeChanged = false; + for (const id of Object.keys(this.aggMap)) { + this.aggMap[id].rangeChanged = false; } } } @@ -146,19 +145,18 @@ export class DataAggregator { private utils: UtilsService, private ignoreDataUpdateOnIntervalTick: boolean) { this.tsKeys.forEach((key) => { - if (!this.dataBuffer[key.agg]) { - this.dataBuffer[key.agg] = {}; + if (!this.dataBuffer[key.id]) { + this.dataBuffer[key.id] = []; } - this.dataBuffer[key.agg][key.key] = []; }); if (this.subsTw.aggregation.stateData) { this.lastPrevKvPairData = {}; } } - private dataBuffer: AggSubscriptionData = {}; - private data: AggSubscriptionData; - private readonly lastPrevKvPairData: {[aggKey: string]: [number, any]}; + private dataBuffer: IndexedSubscriptionData = []; + private data: IndexedSubscriptionData; + private readonly lastPrevKvPairData: {[id: number]: [number, any]}; private aggregationMap: AggregationMap; @@ -201,17 +199,6 @@ export class DataAggregator { } } - private static aggKeyToString(aggKey: AggKey): string { - return `${aggKey.key}_${aggKey.agg}`; - } - - private static aggKeyFromString(aggKeyString: string): AggKey { - const separatorIndex = aggKeyString.lastIndexOf('_'); - const key = aggKeyString.substring(0, separatorIndex); - const agg = AggregationType[aggKeyString.substring(separatorIndex + 1)]; - return { key, agg }; - } - public updateOnDataCb(newOnDataCb: onAggregatedData): onAggregatedData { const prevOnDataCb = this.onDataCb; this.onDataCb = newOnDataCb; @@ -241,7 +228,7 @@ export class DataAggregator { this.aggregationMap = null; } - public onData(data: AggSubscriptionData, update: boolean, history: boolean, detectChanges: boolean) { + public onData(data: IndexedSubscriptionData, update: boolean, history: boolean, detectChanges: boolean) { this.updatedData = true; if (!this.dataReceived || this.resetPending) { let updateIntervalScheduledTime = true; @@ -330,24 +317,24 @@ export class DataAggregator { } } - private updateData(): {[aggType: string]: SubscriptionData} { - this.dataBuffer = {}; + private updateData(): IndexedSubscriptionData { + this.dataBuffer = []; this.tsKeys.forEach((key) => { - if (!this.dataBuffer[key.agg]) { - this.dataBuffer[key.agg] = {}; + if (!this.dataBuffer[key.id]) { + this.dataBuffer[key.id] = []; } - this.dataBuffer[key.agg][key.key] = []; }); - for (const aggKeyString of Object.keys(this.aggregationMap.aggMap)) { - const aggKeyData = this.aggregationMap.aggMap[aggKeyString]; - const aggKey = DataAggregator.aggKeyFromString(aggKeyString); + for (const idStr of Object.keys(this.aggregationMap.aggMap)) { + const id = Number(idStr); + const aggKeyData = this.aggregationMap.aggMap[id]; + const aggKey = this.aggKeyById(id); const noAggregation = aggKey.agg === AggregationType.NONE; - let keyData = this.dataBuffer[aggKey.agg][aggKey.key]; + let keyData = this.dataBuffer[id]; aggKeyData.forEach((aggData, aggTimestamp) => { if (aggTimestamp < this.startTs) { if (this.subsTw.aggregation.stateData && - (!this.lastPrevKvPairData[aggKeyString] || this.lastPrevKvPairData[aggKeyString][0] < aggTimestamp)) { - this.lastPrevKvPairData[aggKeyString] = [aggTimestamp, aggData.aggValue]; + (!this.lastPrevKvPairData[id] || this.lastPrevKvPairData[id][0] < aggTimestamp)) { + this.lastPrevKvPairData[id] = [aggTimestamp, aggData.aggValue]; } aggKeyData.delete(aggTimestamp); this.updatedData = true; @@ -358,12 +345,12 @@ export class DataAggregator { }); keyData.sort((set1, set2) => set1[0] - set2[0]); if (this.subsTw.aggregation.stateData) { - this.updateStateBounds(keyData, deepClone(this.lastPrevKvPairData[aggKeyString])); + this.updateStateBounds(keyData, deepClone(this.lastPrevKvPairData[id])); } if (keyData.length > this.subsTw.aggregation.limit) { keyData = keyData.slice(keyData.length - this.subsTw.aggregation.limit); } - this.dataBuffer[aggKey.agg][aggKey.key] = keyData; + this.dataBuffer[id] = keyData; } return this.dataBuffer; } @@ -396,69 +383,71 @@ export class DataAggregator { } } - private processAggregatedData(data: AggSubscriptionData): AggregationMap { + private processAggregatedData(data: IndexedSubscriptionData): AggregationMap { const aggregationMap = new AggregationMap(); - for (const aggTypeString of Object.keys(data)) { - const aggType = AggregationType[aggTypeString]; + for (const idStr of Object.keys(data)) { + const id = Number(idStr); + const aggKey = this.aggKeyById(id); + const aggType = aggKey.agg; const isCount = aggType === AggregationType.COUNT; const noAggregation = aggType === AggregationType.NONE; - for (const key of Object.keys(data[aggType])) { - const aggKey = DataAggregator.aggKeyToString({key, agg: aggType}); - let aggKeyData = aggregationMap.aggMap[aggKey]; - if (!aggKeyData) { - aggKeyData = new AggDataMap(); - aggregationMap.aggMap[aggKey] = aggKeyData; - } - const keyData = data[aggType][key]; - keyData.forEach((kvPair) => { - const timestamp = kvPair[0]; - const value = DataAggregator.convertValue(kvPair[1], noAggregation); - const tsKey = timestamp; - const aggData = { - count: isCount ? value : isDefinedAndNotNull(kvPair[2]) ? kvPair[2] : 1, - sum: value, - aggValue: value - }; - aggKeyData.set(tsKey, aggData); - }); + let aggKeyData = aggregationMap.aggMap[id]; + if (!aggKeyData) { + aggKeyData = new AggDataMap(); + aggregationMap.aggMap[id] = aggKeyData; } + const keyData = data[id]; + keyData.forEach((kvPair) => { + const timestamp = kvPair[0]; + const value = DataAggregator.convertValue(kvPair[1], noAggregation); + const tsKey = timestamp; + const aggData = { + count: isCount ? value : isDefinedAndNotNull(kvPair[2]) ? kvPair[2] : 1, + sum: value, + aggValue: value + }; + aggKeyData.set(tsKey, aggData); + }); } return aggregationMap; } - private updateAggregatedData(data: AggSubscriptionData) { - for (const aggTypeString of Object.keys(data)) { - const aggType = AggregationType[aggTypeString]; + private updateAggregatedData(data: IndexedSubscriptionData) { + for (const idStr of Object.keys(data)) { + const id = Number(idStr); + const aggKey = this.aggKeyById(id); + const aggType = aggKey.agg; const isCount = aggType === AggregationType.COUNT; const noAggregation = aggType === AggregationType.NONE; - for (const key of Object.keys(data[aggType])) { - const aggKey = DataAggregator.aggKeyToString({key, agg: aggType}); - let aggKeyData = this.aggregationMap.aggMap[aggKey]; - if (!aggKeyData) { - aggKeyData = new AggDataMap(); - this.aggregationMap.aggMap[aggKey] = aggKeyData; - } - const keyData = data[aggType][key]; - keyData.forEach((kvPair) => { - const timestamp = kvPair[0]; - const value = DataAggregator.convertValue(kvPair[1], noAggregation); - const aggTimestamp = noAggregation ? timestamp : (this.startTs + - Math.floor((timestamp - this.startTs) / this.subsTw.aggregation.interval) * - this.subsTw.aggregation.interval + this.subsTw.aggregation.interval / 2); - let aggData = aggKeyData.get(aggTimestamp); - if (!aggData) { - aggData = { - count: isDefinedAndNotNull(kvPair[2]) ? kvPair[2] : 1, - sum: value, - aggValue: isCount ? 1 : value - }; - aggKeyData.set(aggTimestamp, aggData); - } else { - DataAggregator.getAggFunction(aggType)(aggData, value); - } - }); + let aggKeyData = this.aggregationMap.aggMap[id]; + if (!aggKeyData) { + aggKeyData = new AggDataMap(); + this.aggregationMap.aggMap[id] = aggKeyData; } + const keyData = data[id]; + keyData.forEach((kvPair) => { + const timestamp = kvPair[0]; + const value = DataAggregator.convertValue(kvPair[1], noAggregation); + const aggTimestamp = noAggregation ? timestamp : (this.startTs + + Math.floor((timestamp - this.startTs) / this.subsTw.aggregation.interval) * + this.subsTw.aggregation.interval + this.subsTw.aggregation.interval / 2); + let aggData = aggKeyData.get(aggTimestamp); + if (!aggData) { + aggData = { + count: isDefinedAndNotNull(kvPair[2]) ? kvPair[2] : 1, + sum: value, + aggValue: isCount ? 1 : value + }; + aggKeyData.set(aggTimestamp, aggData); + } else { + DataAggregator.getAggFunction(aggType)(aggData, value); + } + }); } } + private aggKeyById(id: number): AggKey { + return this.tsKeys.find(key => key.id === id); + } + } diff --git a/ui-ngx/src/app/core/api/entity-data-subscription.ts b/ui-ngx/src/app/core/api/entity-data-subscription.ts index 8032a32973..75c4e0b922 100644 --- a/ui-ngx/src/app/core/api/entity-data-subscription.ts +++ b/ui-ngx/src/app/core/api/entity-data-subscription.ts @@ -14,9 +14,16 @@ /// limitations under the License. /// -import { DataSet, DataSetHolder, DatasourceType, widgetType } from '@shared/models/widget.models'; -import { AggregationType, getCurrentTime, SubscriptionTimewindow } from '@shared/models/time/time.models'; +import { ComparisonResultType, DataSet, DataSetHolder, DatasourceType, widgetType } from '@shared/models/widget.models'; import { + AggregationType, + ComparisonDuration, + createTimewindowForComparison, + getCurrentTime, + SubscriptionTimewindow +} from '@shared/models/time/time.models'; +import { + ComparisonTsValue, EntityData, EntityDataPageLink, EntityFilter, @@ -29,10 +36,10 @@ import { } from '@shared/models/query/query.models'; import { AggKey, - AggSubscriptionData, DataKeyType, EntityCountCmd, EntityDataCmd, + IndexedSubscriptionData, SubscriptionData, TelemetryService, TelemetrySubscriber @@ -46,7 +53,6 @@ import { NULL_UUID } from '@shared/models/id/has-uuid'; import { EntityType } from '@shared/models/entity-type.models'; import { Observable, of, ReplaySubject, Subject } from 'rxjs'; import { EntityId } from '@shared/models/id/entity-id'; -import _ from 'lodash'; import Timeout = NodeJS.Timeout; declare type DataKeyFunction = (time: number, prevValue: any) => any; @@ -58,6 +64,10 @@ export interface SubscriptionDataKey { name: string; type: DataKeyType; aggregationType?: AggregationType; + comparisonEnabled?: boolean; + timeForComparison?: ComparisonDuration; + comparisonCustomIntervalValue?: number; + comparisonResultType?: ComparisonResultType; funcBody: string; func?: DataKeyFunction; postFuncBody: string; @@ -106,6 +116,7 @@ export class EntityDataSubscription { private tsFields: Array; private latestValues: Array; private aggTsValues: Array; + private aggTsComparisonValues: Array; private entityDataResolveSubject: Subject; private pageData: PageData; @@ -115,6 +126,7 @@ export class EntityDataSubscription { private dataAggregators: Array; private tsLatestDataAggregators: Array; private dataKeys: {[key: string]: Array | SubscriptionDataKey} = {}; + private dataKeysList: SubscriptionDataKey[] = []; private datasourceData: {[index: number]: {[key: string]: DataSetHolder}}; private datasourceOrigData: {[index: number]: {[key: string]: DataSetHolder}}; private entityIdToDataIndex: {[id: string]: number}; @@ -136,9 +148,37 @@ export class EntityDataSubscription { return val; } + private static calculateComparisonValue(key: SubscriptionDataKey, comparisonTsValue: ComparisonTsValue): [number, any, number?][] { + let timestamp: number; + let value: any; + switch (key.comparisonResultType) { + case ComparisonResultType.PREVIOUS_VALUE: + timestamp = comparisonTsValue.previous.ts; + value = comparisonTsValue.previous.value; + break; + case ComparisonResultType.DELTA_ABSOLUTE: + case ComparisonResultType.DELTA_PERCENT: + timestamp = comparisonTsValue.previous.ts; + const currentVal = EntityDataSubscription.convertValue(comparisonTsValue.current.value); + const prevVal = EntityDataSubscription.convertValue(comparisonTsValue.previous.value); + if (isNumeric(currentVal) && isNumeric(prevVal)) { + if (key.comparisonResultType === ComparisonResultType.DELTA_ABSOLUTE) { + value = currentVal - prevVal; + } else { + value = (currentVal - prevVal) / prevVal * 100; + } + } else { + value = ''; + } + break; + } + return [[timestamp, value]]; + } + private initializeSubscription() { for (let i = 0; i < this.entityDataSubscriptionOptions.dataKeys.length; i++) { const dataKey = deepClone(this.entityDataSubscriptionOptions.dataKeys[i]); + this.dataKeysList.push(dataKey); dataKey.index = i; if (this.datasourceType === DatasourceType.function) { if (!dataKey.func) { @@ -156,8 +196,8 @@ export class EntityDataSubscription { if (this.datasourceType === DatasourceType.function) { key = `${dataKey.name}_${dataKey.index}_${dataKey.type}${dataKey.latest ? '_latest' : ''}`; } else { - const aggSuffix = dataKey.aggregationType && dataKey.aggregationType !== AggregationType.NONE ? `_${dataKey.aggregationType.toLowerCase()}` : ''; - key = `${dataKey.name}_${dataKey.type}${aggSuffix}${dataKey.latest ? '_latest' : ''}`; + const keyIndexSuffix = dataKey.aggregationType && dataKey.aggregationType !== AggregationType.NONE ? `_${dataKey.index}` : ''; + key = `${dataKey.name}_${dataKey.type}${keyIndexSuffix}${dataKey.latest ? '_latest' : ''}`; } let dataKeysList = this.dataKeys[key] as Array; if (!dataKeysList) { @@ -213,7 +253,7 @@ export class EntityDataSubscription { } if (this.datasourceType === DatasourceType.entity) { const entityFields: Array = - this.entityDataSubscriptionOptions.dataKeys.filter(dataKey => dataKey.type === DataKeyType.entityField).map( + this.dataKeysList.filter(dataKey => dataKey.type === DataKeyType.entityField).map( dataKey => ({ type: EntityKeyType.ENTITY_FIELD, key: dataKey.name }) ); if (!entityFields.find(key => key.key === 'name')) { @@ -235,18 +275,18 @@ export class EntityDataSubscription { }); } - this.attrFields = this.entityDataSubscriptionOptions.dataKeys.filter(dataKey => dataKey.type === DataKeyType.attribute).map( + this.attrFields = this.dataKeysList.filter(dataKey => dataKey.type === DataKeyType.attribute).map( dataKey => ({ type: EntityKeyType.ATTRIBUTE, key: dataKey.name }) ); - this.tsFields = this.entityDataSubscriptionOptions.dataKeys. + this.tsFields = this.dataKeysList. filter(dataKey => dataKey.type === DataKeyType.timeseries && (!dataKey.aggregationType || dataKey.aggregationType === AggregationType.NONE) && !dataKey.latest).map( dataKey => ({ type: EntityKeyType.TIME_SERIES, key: dataKey.name }) ); if (this.entityDataSubscriptionOptions.type === widgetType.timeseries) { - const latestTsFields = this.entityDataSubscriptionOptions.dataKeys. + const latestTsFields = this.dataKeysList. filter(dataKey => dataKey.type === DataKeyType.timeseries && dataKey.latest && (!dataKey.aggregationType || dataKey.aggregationType === AggregationType.NONE)).map( dataKey => ({ type: EntityKeyType.TIME_SERIES, key: dataKey.name }) @@ -256,12 +296,19 @@ export class EntityDataSubscription { this.latestValues = this.attrFields.concat(this.tsFields); } - this.aggTsValues = this.entityDataSubscriptionOptions.dataKeys. + this.aggTsValues = this.dataKeysList. filter(dataKey => dataKey.type === DataKeyType.timeseries && - dataKey.aggregationType && dataKey.aggregationType !== AggregationType.NONE).map( - dataKey => ({ key: dataKey.name, agg: dataKey.aggregationType }) + dataKey.aggregationType && dataKey.aggregationType !== AggregationType.NONE && !dataKey.comparisonEnabled).map( + dataKey => ({ id: dataKey.index, key: dataKey.name, agg: dataKey.aggregationType }) ); + this.aggTsComparisonValues = this.dataKeysList. + filter(dataKey => dataKey.type === DataKeyType.timeseries && + dataKey.aggregationType && dataKey.aggregationType !== AggregationType.NONE && dataKey.comparisonEnabled).map( + dataKey => ({ id: dataKey.index, key: dataKey.name, agg: dataKey.aggregationType, + previousValueOnly: dataKey.comparisonResultType === ComparisonResultType.PREVIOUS_VALUE }) + ); + this.subscriber = new TelemetrySubscriber(this.telemetryService); this.dataCommand = new EntityDataCmd(); @@ -392,7 +439,7 @@ export class EntityDataSubscription { entityType: null }; - const countKey = this.entityDataSubscriptionOptions.dataKeys[0]; + const countKey = this.dataKeysList[0]; let dataReceived = false; @@ -530,24 +577,30 @@ export class EntityDataSubscription { } else if (this.entityDataSubscriptionOptions.type === widgetType.latest) { latestValuesKeys = this.latestValues; } - if (this.aggTsValues.length > 0) { - if (this.history) { - cmd.aggHistoryCmd = { - keys: this.aggTsValues, - startTs: this.subsTw.fixedWindow.startTimeMs, - endTs: this.subsTw.fixedWindow.endTimeMs - }; - } else if (!this.isFloatingTimewindow) { - cmd.aggTsCmd = { - keys: this.aggTsValues, - startTs: this.subsTw.startTs, - timeWindow: this.subsTw.aggregation.timeWindow - }; - if (latestValuesKeys.length > 0) { - const tsKeys = this.aggTsValues.map(key => key.key); - latestValuesKeys = latestValuesKeys.filter(latestKey => latestKey.type !== EntityKeyType.TIME_SERIES - || !tsKeys.includes(latestKey.key)); - } + if (this.history && (this.aggTsValues.length > 0 || this.aggTsComparisonValues.length > 0)) { + for (const aggTsComparison of this.aggTsComparisonValues) { + const subscriptionDataKey = this.dataKeyByIndex(aggTsComparison.id); + const timewindowForComparison = + createTimewindowForComparison(this.subsTw, subscriptionDataKey.timeForComparison, + subscriptionDataKey.comparisonCustomIntervalValue); + aggTsComparison.previousStartTs = timewindowForComparison.fixedWindow.startTimeMs; + aggTsComparison.previousEndTs = timewindowForComparison.fixedWindow.endTimeMs; + } + cmd.aggHistoryCmd = { + keys: [...this.aggTsValues, ...this.aggTsComparisonValues], + startTs: this.subsTw.fixedWindow.startTimeMs, + endTs: this.subsTw.fixedWindow.endTimeMs + }; + } else if (!this.isFloatingTimewindow && this.aggTsValues.length > 0) { + cmd.aggTsCmd = { + keys: this.aggTsValues, + startTs: this.subsTw.startTs, + timeWindow: this.subsTw.aggregation.timeWindow + }; + if (latestValuesKeys.length > 0) { + const tsKeys = this.aggTsValues.map(key => key.key); + latestValuesKeys = latestValuesKeys.filter(latestKey => latestKey.type !== EntityKeyType.TIME_SERIES + || !tsKeys.includes(latestKey.key)); } } if (latestValuesKeys.length > 0) { @@ -592,29 +645,21 @@ export class EntityDataSubscription { this.resetData(); if (this.entityDataSubscriptionOptions.type === widgetType.timeseries) { - let tsKeyNames: string[] = []; + let tsKeyIds: number[]; if (this.datasourceType === DatasourceType.function) { - for (const key of Object.keys(this.dataKeys)) { - const dataKeysList = this.dataKeys[key] as Array; - dataKeysList.forEach((subscriptionDataKey) => { - if (!subscriptionDataKey.latest) { - tsKeyNames.push(`${subscriptionDataKey.name}_${subscriptionDataKey.index}`); - } - }); - } + tsKeyIds = this.dataKeysList.filter(key => !key.latest).map(key => key.index); } else { - tsKeyNames = this.tsFields ? this.tsFields.map(field => field.key) : []; + tsKeyIds = this.dataKeysList. + filter(dataKey => dataKey.type === DataKeyType.timeseries && + (!dataKey.aggregationType || dataKey.aggregationType === AggregationType.NONE) && !dataKey.latest).map( + dataKey => dataKey.index + ); } - const aggKeys: AggKey[] = tsKeyNames.map(key => ({key, agg: this.subsTw.aggregation.type})); + const aggKeys: AggKey[] = tsKeyIds.map(key => ({id: key, key: key + '', agg: this.subsTw.aggregation.type})); if (aggKeys.length) { for (let dataIndex = 0; dataIndex < this.pageData.data.length; dataIndex++) { - if (this.datasourceType === DatasourceType.function) { - this.dataAggregators[dataIndex] = this.createRealtimeDataAggregator(this.subsTw, aggKeys, - false, DataKeyType.function, dataIndex, this.notifyListener.bind(this)); - } else { - this.dataAggregators[dataIndex] = this.createRealtimeDataAggregator(this.subsTw, aggKeys, - false, DataKeyType.timeseries, dataIndex, this.notifyListener.bind(this)); - } + this.dataAggregators[dataIndex] = this.createRealtimeDataAggregator(this.subsTw, aggKeys, + false, dataIndex, this.notifyListener.bind(this)); } } } @@ -625,34 +670,35 @@ export class EntityDataSubscription { aggLatestTimewindow.aggregation.interval = aggLatestTimewindow.aggregation.timeWindow; for (let dataIndex = 0; dataIndex < this.pageData.data.length; dataIndex++) { this.tsLatestDataAggregators[dataIndex] = this.createRealtimeDataAggregator(aggLatestTimewindow, this.aggTsValues, - true, DataKeyType.timeseries, dataIndex, this.notifyListener.bind(this)); + true, dataIndex, this.notifyListener.bind(this)); } } else { - const tsKeysByAggType = _.groupBy(this.aggTsValues, value => value.agg); - const aggSubscriptionData: AggSubscriptionData = {}; - for (const aggTypeString of Object.keys(tsKeysByAggType)) { - const tsKeys = tsKeysByAggType[aggTypeString]; - const latestTsAggSubsciptionData: SubscriptionData = {}; - for (const tsKey of tsKeys) { - latestTsAggSubsciptionData[tsKey.key] = [[0, 'Not supported!']]; - } - aggSubscriptionData[aggTypeString] = latestTsAggSubsciptionData; - } - for (let dataIndex = 0; dataIndex < this.pageData.data.length; dataIndex++) { - this.onAggData(aggSubscriptionData, DataKeyType.timeseries, dataIndex, true, - this.entityDataSubscriptionOptions.type === widgetType.timeseries, true, - (data, dataIndex1, dataKeyIndex, detectChanges, isLatest) => { - if (!this.data[dataIndex1]) { - this.data[dataIndex1] = []; - } - this.data[dataIndex1][dataKeyIndex] = data; - if (isUpdate) { - this.notifyListener(data, dataIndex1, dataKeyIndex, detectChanges, isLatest); - } - }); - } + this.reportNotSupported(this.aggTsValues, isUpdate); } } + if (!this.history && this.aggTsComparisonValues && this.aggTsComparisonValues.length) { + this.reportNotSupported(this.aggTsComparisonValues, isUpdate); + } + } + + private reportNotSupported(keys: AggKey[], isUpdate: boolean) { + const indexedData: IndexedSubscriptionData = []; + for (const key of keys) { + indexedData[key.id] = [[0, 'Not supported!']]; + } + for (let dataIndex = 0; dataIndex < this.pageData.data.length; dataIndex++) { + this.onIndexedData(indexedData, dataIndex, true, + this.entityDataSubscriptionOptions.type === widgetType.timeseries, + (data, dataIndex1, dataKeyIndex, detectChanges, isLatest) => { + if (!this.data[dataIndex1]) { + this.data[dataIndex1] = []; + } + this.data[dataIndex1][dataKeyIndex] = data; + if (isUpdate) { + this.notifyListener(data, dataIndex1, dataKeyIndex, detectChanges, isLatest); + } + }); + } } private resetData() { @@ -777,20 +823,29 @@ export class EntityDataSubscription { if (this.entityDataSubscriptionOptions.type === widgetType.latest || this.entityDataSubscriptionOptions.type === widgetType.timeseries) { if (entityData.aggLatest) { - if (this.tsLatestDataAggregators && this.tsLatestDataAggregators[dataIndex]) { - const dataAggregator = this.tsLatestDataAggregators[dataIndex]; - const aggSubscriptionData: AggSubscriptionData = {}; - for (const aggTypeString of Object.keys(entityData.aggLatest)) { - aggSubscriptionData[aggTypeString] = this.toSubscriptionData(entityData.aggLatest[aggTypeString], false); + const aggData: IndexedSubscriptionData = []; + for (const idStr of Object.keys(entityData.aggLatest)) { + const id = Number(idStr); + const dataKey = this.dataKeyByIndex(id); + const aggLatestData = entityData.aggLatest[id]; + if (dataKey.comparisonEnabled) { + const keyData = EntityDataSubscription.calculateComparisonValue(dataKey, aggLatestData); + this.onKeyData(keyData, dataKey.name, id, dataKey.type, dataIndex, true, + this.entityDataSubscriptionOptions.type === widgetType.timeseries, true, dataUpdatedCb); + } else { + aggData[id] = [[aggLatestData.current.ts, aggLatestData.current.value, aggLatestData.current.count]]; } + } + if (Object.keys(aggData).length > 0 && this.tsLatestDataAggregators && this.tsLatestDataAggregators[dataIndex]) { + const dataAggregator = this.tsLatestDataAggregators[dataIndex]; let prevDataCb; if (!isUpdate) { prevDataCb = dataAggregator.updateOnDataCb((data, detectChanges) => { - this.onAggData(data, DataKeyType.timeseries, dataIndex, detectChanges, - this.entityDataSubscriptionOptions.type === widgetType.timeseries, true, dataUpdatedCb); + this.onIndexedData(data, dataIndex, detectChanges, + this.entityDataSubscriptionOptions.type === widgetType.timeseries, dataUpdatedCb); }); } - dataAggregator.onData(aggSubscriptionData, false, this.history, true); + dataAggregator.onData(aggData, false, this.history, true); if (prevDataCb) { dataAggregator.updateOnDataCb(prevDataCb); } @@ -809,26 +864,20 @@ export class EntityDataSubscription { latestTsSubsciptionData[latestTsKey.key] = subscriptionData[latestTsKey.key]; } this.onData(latestTsSubsciptionData, dataKeyType, dataIndex, true, - this.entityDataSubscriptionOptions.type === widgetType.timeseries, false, dataUpdatedCb); + this.entityDataSubscriptionOptions.type === widgetType.timeseries, dataUpdatedCb); } const aggTsKeys = this.aggTsValues.filter(key => keys.includes(key.key)); if (!this.history && aggTsKeys.length && this.tsLatestDataAggregators && this.tsLatestDataAggregators[dataIndex]) { const dataAggregator = this.tsLatestDataAggregators[dataIndex]; - const tsKeysByAggType = _.groupBy(aggTsKeys, value => value.agg); - const aggSubscriptionData: AggSubscriptionData = {}; - for (const aggTypeString of Object.keys(tsKeysByAggType)) { - const tsKeys = tsKeysByAggType[aggTypeString]; - const latestTsAggSubsciptionData: SubscriptionData = {}; - for (const tsKey of tsKeys) { - latestTsAggSubsciptionData[tsKey.key] = subscriptionData[tsKey.key]; - } - aggSubscriptionData[aggTypeString] = latestTsAggSubsciptionData; + const indexedData: IndexedSubscriptionData = []; + for (const aggKey of aggTsKeys) { + indexedData[aggKey.id] = subscriptionData[aggKey.key]; } - dataAggregator.onData(aggSubscriptionData, true, false, true); + dataAggregator.onData(indexedData, true, false, true); } } else { this.onData(subscriptionData, dataKeyType, dataIndex, true, - this.entityDataSubscriptionOptions.type === widgetType.timeseries, false, dataUpdatedCb); + this.entityDataSubscriptionOptions.type === widgetType.timeseries, dataUpdatedCb); } } } @@ -837,100 +886,116 @@ export class EntityDataSubscription { const subscriptionData = this.toSubscriptionData(entityData.timeseries, true); if (this.dataAggregators && this.dataAggregators[dataIndex]) { const dataAggregator = this.dataAggregators[dataIndex]; - const aggSubscriptionData: AggSubscriptionData = {}; - aggSubscriptionData[this.subsTw.aggregation.type] = subscriptionData; + const keyNames = Object.keys(subscriptionData); + const dataKeys = this.timeseriesDataKeysByKeyNames(keyNames); + const indexedData: IndexedSubscriptionData = []; + for (const dataKey of dataKeys) { + indexedData[dataKey.index] = subscriptionData[dataKey.name]; + } let prevDataCb; if (!isUpdate) { prevDataCb = dataAggregator.updateOnDataCb((data, detectChanges) => { - this.onAggData(data, this.datasourceType === DatasourceType.function ? - DataKeyType.function : DataKeyType.timeseries, dataIndex, detectChanges, false, false, dataUpdatedCb); + this.onIndexedData(data, dataIndex, detectChanges, false, dataUpdatedCb); }); } - dataAggregator.onData(aggSubscriptionData, false, this.history, true); + dataAggregator.onData(indexedData, false, this.history, true); if (prevDataCb) { dataAggregator.updateOnDataCb(prevDataCb); } } else if (!this.history && !isUpdate) { - this.onData(subscriptionData, DataKeyType.timeseries, dataIndex, true, false, false, dataUpdatedCb); + this.onData(subscriptionData, DataKeyType.timeseries, dataIndex, true, false, dataUpdatedCb); } } } private onData(sourceData: SubscriptionData, type: DataKeyType, dataIndex: number, detectChanges: boolean, - isTsLatest: boolean, isAggLatest: boolean, dataUpdatedCb: DataUpdatedCb) { - const aggSubscriptionData: AggSubscriptionData = {}; - aggSubscriptionData[AggregationType.NONE] = sourceData; - this.onAggData(aggSubscriptionData, type, dataIndex, detectChanges, isTsLatest, isAggLatest, dataUpdatedCb); + isTsLatest: boolean, dataUpdatedCb: DataUpdatedCb) { + for (const key of Object.keys(sourceData)) { + const keyData = sourceData[key]; + this.onKeyData(keyData, key, 0, type, + dataIndex, detectChanges, isTsLatest, false, dataUpdatedCb); + } + } + + private onIndexedData(sourceData: IndexedSubscriptionData, dataIndex: number, detectChanges: boolean, + isTsLatest: boolean, dataUpdatedCb: DataUpdatedCb) { + for (const indexStr of Object.keys(sourceData)) { + const id = Number(indexStr); + const dataKey = this.dataKeyByIndex(id); + const isAggLatest = dataKey.aggregationType && dataKey.aggregationType !== AggregationType.NONE; + const keyData = sourceData[id]; + let keyName = dataKey.name; + if (dataKey.type === DataKeyType.function) { + keyName += `_${dataKey.index}`; + } + this.onKeyData(keyData, keyName, id, dataKey.type, + dataIndex, detectChanges, isTsLatest, isAggLatest, dataUpdatedCb); + } } - private onAggData(sourceData: AggSubscriptionData, type: DataKeyType, dataIndex: number, detectChanges: boolean, + private onKeyData(keyData: [number, any, number?][], keyName: string, id: number, type: DataKeyType, + dataIndex: number, detectChanges: boolean, isTsLatest: boolean, isAggLatest: boolean, dataUpdatedCb: DataUpdatedCb) { - for (const aggTypeString of Object.keys(sourceData)) { - const aggType = AggregationType[aggTypeString]; - const aggSuffix = isAggLatest ? (aggType !== AggregationType.NONE ? `_${aggType.toLowerCase()}` : '') : ''; - for (const keyName of Object.keys(sourceData[aggType])) { - const keyData = sourceData[aggType][keyName]; - const key = `${keyName}_${type}${aggSuffix}${isTsLatest ? '_latest' : ''}`; - const dataKeyList = this.dataKeys[key] as Array; - for (let keyIndex = 0; dataKeyList && keyIndex < dataKeyList.length; keyIndex++) { - const datasourceKey = `${key}_${keyIndex}`; - if (this.datasourceData[dataIndex][datasourceKey].data) { - const dataKey = dataKeyList[keyIndex]; - const data: DataSet = []; - let prevSeries: [number, any]; - let prevOrigSeries: [number, any]; - let datasourceKeyData: DataSet; - let datasourceOrigKeyData: DataSet; - let update = false; - if (this.realtime && !isTsLatest) { - datasourceKeyData = []; - datasourceOrigKeyData = []; - } else { - datasourceKeyData = this.datasourceData[dataIndex][datasourceKey].data; - datasourceOrigKeyData = this.datasourceOrigData[dataIndex][datasourceKey].data; - } - if (datasourceKeyData.length > 0) { - prevSeries = datasourceKeyData[datasourceKeyData.length - 1]; - prevOrigSeries = datasourceOrigKeyData[datasourceOrigKeyData.length - 1]; - } else { - prevSeries = [0, 0]; - prevOrigSeries = [0, 0]; - } - this.datasourceOrigData[dataIndex][datasourceKey].data = []; - if (this.entityDataSubscriptionOptions.type === widgetType.timeseries && !isTsLatest) { - keyData.forEach((keySeries) => { - let series = keySeries; - const time = series[0]; - this.datasourceOrigData[dataIndex][datasourceKey].data.push([series[0], series[1]]); - let value = EntityDataSubscription.convertValue(series[1]); - if (dataKey.postFunc) { - value = dataKey.postFunc(time, value, prevSeries[1], prevOrigSeries[0], prevOrigSeries[1]); - } - prevOrigSeries = [series[0], series[1]]; - series = [series[0], value]; - data.push([series[0], series[1]]); - prevSeries = [series[0], series[1]]; - }); - update = true; - } else if (this.entityDataSubscriptionOptions.type === widgetType.latest || isTsLatest) { - if (keyData.length > 0) { - let series = keyData[0]; - const time = series[0]; - this.datasourceOrigData[dataIndex][datasourceKey].data.push([series[0], series[1]]); - let value = EntityDataSubscription.convertValue(series[1]); - if (dataKey.postFunc) { - value = dataKey.postFunc(time, value, prevSeries[1], prevOrigSeries[0], prevOrigSeries[1]); - } - series = [time, value]; - data.push([series[0], series[1]]); - } - update = true; + const keyIdSuffix = isAggLatest ? `_${id}` : ''; + const key = `${keyName}_${type}${keyIdSuffix}${isTsLatest ? '_latest' : ''}`; + const dataKeyList = this.dataKeys[key] as Array; + for (let keyIndex = 0; dataKeyList && keyIndex < dataKeyList.length; keyIndex++) { + const datasourceKey = `${key}_${keyIndex}`; + if (this.datasourceData[dataIndex][datasourceKey].data) { + const dataKey = dataKeyList[keyIndex]; + const data: DataSet = []; + let prevSeries: [number, any]; + let prevOrigSeries: [number, any]; + let datasourceKeyData: DataSet; + let datasourceOrigKeyData: DataSet; + let update = false; + if (this.realtime && !isTsLatest) { + datasourceKeyData = []; + datasourceOrigKeyData = []; + } else { + datasourceKeyData = this.datasourceData[dataIndex][datasourceKey].data; + datasourceOrigKeyData = this.datasourceOrigData[dataIndex][datasourceKey].data; + } + if (datasourceKeyData.length > 0) { + prevSeries = datasourceKeyData[datasourceKeyData.length - 1]; + prevOrigSeries = datasourceOrigKeyData[datasourceOrigKeyData.length - 1]; + } else { + prevSeries = [0, 0]; + prevOrigSeries = [0, 0]; + } + this.datasourceOrigData[dataIndex][datasourceKey].data = []; + if (this.entityDataSubscriptionOptions.type === widgetType.timeseries && !isTsLatest) { + keyData.forEach((keySeries) => { + let series = keySeries; + const time = series[0]; + this.datasourceOrigData[dataIndex][datasourceKey].data.push([series[0], series[1]]); + let value = EntityDataSubscription.convertValue(series[1]); + if (dataKey.postFunc) { + value = dataKey.postFunc(time, value, prevSeries[1], prevOrigSeries[0], prevOrigSeries[1]); } - if (update) { - this.datasourceData[dataIndex][datasourceKey].data = data; - dataUpdatedCb(this.datasourceData[dataIndex][datasourceKey], dataIndex, dataKey.index, detectChanges, isTsLatest); + prevOrigSeries = [series[0], series[1]]; + series = [series[0], value]; + data.push([series[0], series[1]]); + prevSeries = [series[0], series[1]]; + }); + update = true; + } else if (this.entityDataSubscriptionOptions.type === widgetType.latest || isTsLatest) { + if (keyData.length > 0) { + let series = keyData[0]; + const time = series[0]; + this.datasourceOrigData[dataIndex][datasourceKey].data.push([series[0], series[1]]); + let value = EntityDataSubscription.convertValue(series[1]); + if (dataKey.postFunc) { + value = dataKey.postFunc(time, value, prevSeries[1], prevOrigSeries[0], prevOrigSeries[1]); } + series = [time, value]; + data.push([series[0], series[1]]); } + update = true; + } + if (update) { + this.datasourceData[dataIndex][datasourceKey].data = data; + dataUpdatedCb(this.datasourceData[dataIndex][datasourceKey], dataIndex, dataKey.index, detectChanges, isTsLatest); } } } @@ -957,13 +1022,12 @@ export class EntityDataSubscription { private createRealtimeDataAggregator(subsTw: SubscriptionTimewindow, tsKeys: Array, isLatestDataAgg: boolean, - dataKeyType: DataKeyType, dataIndex: number, dataUpdatedCb: DataUpdatedCb): DataAggregator { return new DataAggregator( (data, detectChanges) => { - this.onAggData(data, dataKeyType, dataIndex, detectChanges, - isLatestDataAgg && (this.entityDataSubscriptionOptions.type === widgetType.timeseries), isLatestDataAgg, dataUpdatedCb); + this.onIndexedData(data, dataIndex, detectChanges, + isLatestDataAgg && (this.entityDataSubscriptionOptions.type === widgetType.timeseries), dataUpdatedCb); }, tsKeys, isLatestDataAgg, @@ -973,6 +1037,20 @@ export class EntityDataSubscription { ); } + private dataKeyByIndex(index: number): SubscriptionDataKey { + return this.dataKeysList.find(key => key.index === index); + } + + private timeseriesDataKeysByKeyNames(keyNames: string[]): SubscriptionDataKey[] { + const result: SubscriptionDataKey[] = []; + for (const keyName of keyNames) { + const key = `${keyName}_${DataKeyType.timeseries}`; + const dataKeyList = this.dataKeys[key] as Array; + result.push(...dataKeyList); + } + return result; + } + private generateSeries(dataKey: SubscriptionDataKey, startTime: number, endTime: number): [number, any][] { const data: [number, any][] = []; let prevSeries: [number, any]; @@ -1051,9 +1129,7 @@ export class EntityDataSubscription { let startTime: number; let endTime: number; let delta: number; - const aggType = this.entityDataSubscriptionOptions.subscriptionTimewindow.aggregation.type; - const generatedData: AggSubscriptionData = {}; - generatedData[aggType] = {}; + const generatedData: IndexedSubscriptionData = []; if (!this.history) { delta = Math.floor(this.tickElapsed / this.frequency); } @@ -1086,7 +1162,7 @@ export class EntityDataSubscription { endTime = Math.min(currentTime, endTime); } } - generatedData[aggType][`${dataKey.name}_${dataKey.index}`] = this.generateSeries(dataKey, startTime, endTime); + generatedData[dataKey.index] = this.generateSeries(dataKey, startTime, endTime); } if (this.dataAggregators && this.dataAggregators.length) { this.dataAggregators[0].onData(generatedData, true, this.history, detectChanges); diff --git a/ui-ngx/src/app/core/api/entity-data.service.ts b/ui-ngx/src/app/core/api/entity-data.service.ts index 4557d96b0f..6d8ea94dc0 100644 --- a/ui-ngx/src/app/core/api/entity-data.service.ts +++ b/ui-ngx/src/app/core/api/entity-data.service.ts @@ -74,6 +74,21 @@ export class EntityDataService { } } + private static toSubscriptionDataKey(dataKey: DataKey, latest: boolean): SubscriptionDataKey { + return { + name: dataKey.name, + type: dataKey.type, + aggregationType: dataKey.aggregationType, + comparisonEnabled: dataKey.comparisonEnabled, + timeForComparison: dataKey.timeForComparison, + comparisonCustomIntervalValue: dataKey.comparisonCustomIntervalValue, + comparisonResultType: dataKey.comparisonResultType, + funcBody: dataKey.funcBody, + postFuncBody: dataKey.postFuncBody, + latest + }; + } + public prepareSubscription(listener: EntityDataListener, ignoreDataUpdateOnIntervalTick = false): Observable { const datasource = listener.configDatasource; @@ -147,11 +162,11 @@ export class EntityDataService { ignoreDataUpdateOnIntervalTick: boolean): EntityDataSubscriptionOptions { const subscriptionDataKeys: Array = []; datasource.dataKeys.forEach((dataKey) => { - subscriptionDataKeys.push(this.toSubscriptionDataKey(dataKey, false)); + subscriptionDataKeys.push(EntityDataService.toSubscriptionDataKey(dataKey, false)); }); if (datasource.latestDataKeys) { datasource.latestDataKeys.forEach((dataKey) => { - subscriptionDataKeys.push(this.toSubscriptionDataKey(dataKey, true)); + subscriptionDataKeys.push(EntityDataService.toSubscriptionDataKey(dataKey, true)); }); } const entityDataSubscriptionOptions: EntityDataSubscriptionOptions = { @@ -172,15 +187,4 @@ export class EntityDataService { entityDataSubscriptionOptions.ignoreDataUpdateOnIntervalTick = ignoreDataUpdateOnIntervalTick; return entityDataSubscriptionOptions; } - - private toSubscriptionDataKey(dataKey: DataKey, latest: boolean): SubscriptionDataKey { - return { - name: dataKey.name, - type: dataKey.type, - aggregationType: dataKey.aggregationType, - funcBody: dataKey.funcBody, - postFuncBody: dataKey.postFuncBody, - latest - }; - } } diff --git a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html index 8cc9cb4ff3..fe4df7ca62 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html @@ -62,15 +62,69 @@ - - datakey.aggregation-type - - - {{ (aggregation === aggregationTypes.NONE ? 'datakey.latest-value' : aggregationTypesTranslations.get(aggregationTypes[aggregation])) | translate }} - - - {{ dataKeyFormGroup.get('aggregationType').value ? (dataKeyAggregationTypeHintTranslations.get(aggregationTypes[dataKeyFormGroup.get('aggregationType').value]) | translate) : '' }} - +
+ + datakey.aggregation-type + + + {{ (aggregation === aggregationTypes.NONE ? 'datakey.latest-value' : aggregationTypesTranslations.get(aggregationTypes[aggregation])) | translate }} + + + {{ dataKeyFormGroup.get('aggregationType').value ? (dataKeyAggregationTypeHintTranslations.get(aggregationTypes[dataKeyFormGroup.get('aggregationType').value]) | translate) : '' }} + +
+ widgets.chart.comparison-settings + + + + + {{ 'widgets.chart.enable-comparison' | translate }} + + + + +
+ + widgets.chart.time-for-comparison + + + {{ 'widgets.chart.time-for-comparison-previous-interval' | translate }} + + + {{ 'widgets.chart.time-for-comparison-days' | translate }} + + + {{ 'widgets.chart.time-for-comparison-weeks' | translate }} + + + {{ 'widgets.chart.time-for-comparison-months' | translate }} + + + {{ 'widgets.chart.time-for-comparison-years' | translate }} + + + {{ 'widgets.chart.time-for-comparison-custom-interval' | translate }} + + + + + widgets.chart.custom-interval-value + + + + datakey.comparison-result + + + {{ comparisonResultTypeTranslations.get(comparisonResultTypes[comparisonResultType]) | translate }} + + + +
+ + +
+
datakey.data-generation-func
diff --git a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.scss b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.scss index 8b71ee3664..d0d7e56137 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.scss +++ b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.scss @@ -28,6 +28,36 @@ padding-left: 12px; } } + + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + + &.fields-group-slider { + padding: 0; + + legend { + margin-left: 16px; + } + + > .tb-settings { + margin-top: 0; + padding: 0 16px 8px; + } + } + } } } @@ -42,5 +72,49 @@ } } } + .mat-expansion-panel { + &.tb-settings { + box-shadow: none; + + .mat-content { + overflow: visible; + } + + .mat-expansion-panel-header { + padding: 0; + color: rgba(0, 0, 0, 0.87); + + &:hover { + background: none; + } + + .mat-expansion-indicator { + padding: 2px; + } + } + + .mat-expansion-panel-header-description { + align-items: center; + } + + > .mat-expansion-panel-content { + > .mat-expansion-panel-body { + padding: 0; + } + } + } + + .mat-expansion-panel-content { + font: inherit; + } + } + + .mat-slide { + margin: 8px 0; + } + + .mat-slide-toggle-content { + white-space: normal; + } } } diff --git a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.ts index 4085e5dca2..a461796111 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.ts @@ -18,7 +18,13 @@ import { Component, ElementRef, forwardRef, Input, OnInit, ViewChild } from '@an import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { DataKey, dataKeyAggregationTypeHintTranslationMap, Widget, widgetType } from '@shared/models/widget.models'; +import { + ComparisonResultType, comparisonResultTypeTranslationMap, + DataKey, + dataKeyAggregationTypeHintTranslationMap, + Widget, + widgetType +} from '@shared/models/widget.models'; import { ControlValueAccessor, FormBuilder, @@ -43,7 +49,7 @@ import { JsonFormComponentData } from '@shared/components/json-form/json-form-co import { WidgetService } from '@core/http/widget.service'; import { Dashboard } from '@shared/models/dashboard.models'; import { IAliasController } from '@core/api/widget-api.models'; -import { aggregationTranslations, AggregationType } from '@shared/models/time/time.models'; +import { aggregationTranslations, AggregationType, ComparisonDuration } from '@shared/models/time/time.models'; @Component({ selector: 'tb-data-key-config', @@ -76,6 +82,12 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con dataKeyAggregationTypeHintTranslations = dataKeyAggregationTypeHintTranslationMap; + comparisonResultTypes = ComparisonResultType; + + comparisonResults = Object.keys(ComparisonResultType); + + comparisonResultTypeTranslations = comparisonResultTypeTranslationMap; + @Input() entityAliasId: string; @@ -170,6 +182,10 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con this.dataKeyFormGroup = this.fb.group({ name: [null, []], aggregationType: [null, []], + comparisonEnabled: [null, []], + timeForComparison: [null, [Validators.required]], + comparisonCustomIntervalValue: [null, [Validators.required, Validators.min(1000)]], + comparisonResultType: [null, [Validators.required]], label: [null, [Validators.required]], color: [null, [Validators.required]], units: [null, []], @@ -189,6 +205,19 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con } this.dataKeyFormGroup.get('label').patchValue(newLabel); } + this.updateComparisonValidators(); + } + ); + + this.dataKeyFormGroup.get('comparisonEnabled').valueChanges.subscribe( + () => { + this.updateComparisonValues(); + } + ); + + this.dataKeyFormGroup.get('timeForComparison').valueChanges.subscribe( + () => { + this.updateComparisonValues(); } ); @@ -231,21 +260,82 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con this.modelValue.aggregationType = AggregationType.NONE; } this.dataKeyFormGroup.patchValue(this.modelValue, {emitEvent: false}); + this.updateValidators(); + if (this.displayAdvanced) { + this.dataKeySettingsData.model = this.modelValue.settings; + this.dataKeySettingsFormGroup.patchValue({ + settings: this.dataKeySettingsData + }, {emitEvent: false}); + } + } + + private updateValidators() { this.dataKeyFormGroup.get('name').setValidators(this.modelValue.type !== DataKeyType.function && - this.modelValue.type !== DataKeyType.count - ? [Validators.required] : []); + this.modelValue.type !== DataKeyType.count + ? [Validators.required] : []); if (this.modelValue.type === DataKeyType.count) { this.dataKeyFormGroup.get('name').disable({emitEvent: false}); } else { this.dataKeyFormGroup.get('name').enable({emitEvent: false}); } this.dataKeyFormGroup.get('name').updateValueAndValidity({emitEvent: false}); - if (this.displayAdvanced) { - this.dataKeySettingsData.model = this.modelValue.settings; - this.dataKeySettingsFormGroup.patchValue({ - settings: this.dataKeySettingsData - }, {emitEvent: false}); + this.updateComparisonValidators(); + } + + private updateComparisonValues() { + const comparisonEnabled = this.dataKeyFormGroup.get('comparisonEnabled').value; + if (comparisonEnabled) { + const timeForComparison: ComparisonDuration = this.dataKeyFormGroup.get('timeForComparison').value; + if (!timeForComparison) { + this.dataKeyFormGroup.get('timeForComparison').patchValue('previousInterval', {emitEvent: false}); + } else if (timeForComparison === 'customInterval') { + const comparisonCustomIntervalValue = this.dataKeyFormGroup.get('comparisonCustomIntervalValue').value; + if (!comparisonCustomIntervalValue) { + this.dataKeyFormGroup.get('comparisonCustomIntervalValue').patchValue(7200000, {emitEvent: false}); + } + } + const comparisonResultType: ComparisonResultType = this.dataKeyFormGroup.get('comparisonResultType').value; + if (!comparisonResultType) { + this.dataKeyFormGroup.get('comparisonResultType').patchValue(ComparisonResultType.DELTA_ABSOLUTE, {emitEvent: false}); + } + } + this.updateComparisonValidators(); + } + + private updateComparisonValidators() { + const aggregationType: AggregationType = this.dataKeyFormGroup.get('aggregationType').value; + if (aggregationType && aggregationType !== AggregationType.NONE) { + this.dataKeyFormGroup.get('comparisonEnabled').enable({emitEvent: false}); + const comparisonEnabled = this.dataKeyFormGroup.get('comparisonEnabled').value; + if (comparisonEnabled) { + this.dataKeyFormGroup.get('timeForComparison').enable({emitEvent: false}); + const timeForComparison: ComparisonDuration = this.dataKeyFormGroup.get('timeForComparison').value; + if (timeForComparison) { + this.dataKeyFormGroup.get('comparisonResultType').enable({emitEvent: false}); + if (timeForComparison === 'customInterval') { + this.dataKeyFormGroup.get('comparisonCustomIntervalValue').enable({emitEvent: false}); + } else { + this.dataKeyFormGroup.get('comparisonCustomIntervalValue').disable({emitEvent: false}); + } + } else { + this.dataKeyFormGroup.get('comparisonResultType').disable({emitEvent: false}); + this.dataKeyFormGroup.get('comparisonCustomIntervalValue').disable({emitEvent: false}); + } + } else { + this.dataKeyFormGroup.get('timeForComparison').disable({emitEvent: false}); + this.dataKeyFormGroup.get('comparisonResultType').disable({emitEvent: false}); + this.dataKeyFormGroup.get('comparisonCustomIntervalValue').disable({emitEvent: false}); + } + } else { + this.dataKeyFormGroup.get('comparisonEnabled').disable({emitEvent: false}); + this.dataKeyFormGroup.get('timeForComparison').disable({emitEvent: false}); + this.dataKeyFormGroup.get('comparisonResultType').disable({emitEvent: false}); + this.dataKeyFormGroup.get('comparisonCustomIntervalValue').disable({emitEvent: false}); } + this.dataKeyFormGroup.get('comparisonEnabled').updateValueAndValidity({emitEvent: false}); + this.dataKeyFormGroup.get('timeForComparison').updateValueAndValidity({emitEvent: false}); + this.dataKeyFormGroup.get('comparisonResultType').updateValueAndValidity({emitEvent: false}); + this.dataKeyFormGroup.get('comparisonCustomIntervalValue').updateValueAndValidity({emitEvent: false}); } private updateModel() { diff --git a/ui-ngx/src/app/shared/models/query/query.models.ts b/ui-ngx/src/app/shared/models/query/query.models.ts index 95c756f7f9..c1bf255cd3 100644 --- a/ui-ngx/src/app/shared/models/query/query.models.ts +++ b/ui-ngx/src/app/shared/models/query/query.models.ts @@ -763,11 +763,16 @@ export interface TsValue { count?: number; } +export interface ComparisonTsValue { + current?: TsValue; + previous?: TsValue; +} + export interface EntityData { entityId: EntityId; latest: {[entityKeyType: string]: {[key: string]: TsValue}}; timeseries: {[key: string]: Array}; - aggLatest?: {[aggType: string]: {[key: string]: TsValue}}; + aggLatest?: {[id: number]: ComparisonTsValue}; } export interface AlarmData extends AlarmInfo { diff --git a/ui-ngx/src/app/shared/models/telemetry/telemetry.models.ts b/ui-ngx/src/app/shared/models/telemetry/telemetry.models.ts index 04d98567cf..5fef4b832e 100644 --- a/ui-ngx/src/app/shared/models/telemetry/telemetry.models.ts +++ b/ui-ngx/src/app/shared/models/telemetry/telemetry.models.ts @@ -174,8 +174,12 @@ export interface TimeSeriesCmd { } export interface AggKey { + id: number; key: string; agg: AggregationType; + previousStartTs?: number; + previousEndTs?: number; + previousValueOnly?: boolean; } export interface AggEntityHistoryCmd { @@ -314,8 +318,8 @@ export interface SubscriptionData { [key: string]: [number, any, number?][]; } -export interface AggSubscriptionData { - [aggType: string]: SubscriptionData; +export interface IndexedSubscriptionData { + [id: number]: [number, any, number?][]; } export interface SubscriptionDataHolder { diff --git a/ui-ngx/src/app/shared/models/widget.models.ts b/ui-ngx/src/app/shared/models/widget.models.ts index 9efb47758d..c654c87784 100644 --- a/ui-ngx/src/app/shared/models/widget.models.ts +++ b/ui-ngx/src/app/shared/models/widget.models.ts @@ -17,7 +17,7 @@ import { BaseData } from '@shared/models/base-data'; import { TenantId } from '@shared/models/id/tenant-id'; import { WidgetTypeId } from '@shared/models/id/widget-type-id'; -import { AggregationType, Timewindow } from '@shared/models/time/time.models'; +import { AggregationType, ComparisonDuration, Timewindow } from '@shared/models/time/time.models'; import { EntityType } from '@shared/models/entity-type.models'; import { AlarmSearchStatus, AlarmSeverity } from '@shared/models/alarm.models'; import { DataKeyType } from './telemetry/telemetry.models'; @@ -259,9 +259,27 @@ export function defaultLegendConfig(wType: widgetType): LegendConfig { }; } +export enum ComparisonResultType { + PREVIOUS_VALUE = 'PREVIOUS_VALUE', + DELTA_ABSOLUTE = 'DELTA_ABSOLUTE', + DELTA_PERCENT = 'DELTA_PERCENT' +} + +export const comparisonResultTypeTranslationMap = new Map( + [ + [ComparisonResultType.PREVIOUS_VALUE, 'datakey.comparison-result-previous-value'], + [ComparisonResultType.DELTA_ABSOLUTE, 'datakey.comparison-result-delta-absolute'], + [ComparisonResultType.DELTA_PERCENT, 'datakey.comparison-result-delta-percent'] + ] +); + export interface KeyInfo { name: string; aggregationType?: AggregationType; + comparisonEnabled?: boolean; + timeForComparison?: ComparisonDuration; + comparisonCustomIntervalValue?: number; + comparisonResultType?: ComparisonResultType; label?: string; color?: string; funcBody?: string; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index d07b262bcc..e64d7b9b78 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1047,7 +1047,11 @@ "aggregation-type-max-hint": "Take max value", "aggregation-type-avg-hint": "Calculate average value", "aggregation-type-sum-hint": "Calculate sum value", - "aggregation-type-count-hint": "Calculate count value" + "aggregation-type-count-hint": "Calculate count value", + "comparison-result": "Comparison result", + "comparison-result-previous-value": "Previous value", + "comparison-result-delta-absolute": "Delta (absolute)", + "comparison-result-delta-percent": "Delta (percent)" }, "datasource": { "type": "Datasource type", From 70466bf71ca68e38b04239c60d0d13baa49beb66 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 15 Sep 2022 14:18:16 +0300 Subject: [PATCH 15/18] UI: Improve timewindow configuration for aggregation comparison --- .../app/core/api/entity-data-subscription.ts | 2 +- .../widget/widget-config.component.html | 2 + .../widget/widget-config.component.ts | 11 ++++- .../widget/widget-container.component.html | 2 + .../home/models/dashboard-component.models.ts | 9 ++++- .../components/time/timewindow.component.ts | 40 +++++++++++++++++-- ui-ngx/src/app/shared/models/widget.models.ts | 17 ++++++++ 7 files changed, 77 insertions(+), 6 deletions(-) diff --git a/ui-ngx/src/app/core/api/entity-data-subscription.ts b/ui-ngx/src/app/core/api/entity-data-subscription.ts index 75c4e0b922..4dc0e7c4e1 100644 --- a/ui-ngx/src/app/core/api/entity-data-subscription.ts +++ b/ui-ngx/src/app/core/api/entity-data-subscription.ts @@ -1033,7 +1033,7 @@ export class EntityDataSubscription { isLatestDataAgg, subsTw, this.utils, - this.entityDataSubscriptionOptions.ignoreDataUpdateOnIntervalTick + this.entityDataSubscriptionOptions.ignoreDataUpdateOnIntervalTick || isLatestDataAgg ); } diff --git a/ui-ngx/src/app/modules/home/components/widget/widget-config.component.html b/ui-ngx/src/app/modules/home/components/widget/widget-config.component.html index 1659ee636d..3adb2d4fb6 100644 --- a/ui-ngx/src/app/modules/home/components/widget/widget-config.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/widget-config.component.html @@ -34,6 +34,8 @@ style="padding-right: 8px;">widget-config.timewindow diff --git a/ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts index 7d82a6ca02..2286510ba3 100644 --- a/ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts @@ -20,7 +20,7 @@ import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { DataKey, - Datasource, datasourcesHasAggregation, + Datasource, datasourcesHasAggregation, datasourcesHasOnlyComparisonAggregation, DatasourceType, datasourceTypeTranslationMap, defaultLegendConfig, @@ -743,6 +743,15 @@ export class WidgetConfigComponent extends PageComponent implements OnInit, Cont } } + public onlyHistoryTimewindow(): boolean { + if (this.widgetType === widgetType.latest) { + const datasources = this.dataSettings.get('datasources').value; + return datasourcesHasOnlyComparisonAggregation(datasources); + } else { + return false; + } + } + public onDatasourceDrop(event: CdkDragDrop) { const datasourcesFormArray = this.datasourcesFormArray(); const datasourceForm = datasourcesFormArray.at(event.previousIndex); diff --git a/ui-ngx/src/app/modules/home/components/widget/widget-container.component.html b/ui-ngx/src/app/modules/home/components/widget/widget-container.component.html index 9f99ffbf29..ad8f65a48b 100644 --- a/ui-ngx/src/app/modules/home/components/widget/widget-container.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/widget-container.component.html @@ -47,6 +47,8 @@ { + this.notifyChanged(); + }); + } else { + this.updateDisplayValue(); + } } notifyChanged() { @@ -309,7 +343,7 @@ export class TimewindowComponent implements OnInit, OnDestroy, ControlValueAcces this.millisecondsToTimeStringPipe.transform(this.innerValue.realtime.timewindowMs); } } else { - this.innerValue.displayValue = !this.historyOnly ? (this.translate.instant('timewindow.history') + ' - ') : ''; + this.innerValue.displayValue = (!this.historyOnly || this.alwaysDisplayTypePrefix) ? (this.translate.instant('timewindow.history') + ' - ') : ''; if (this.innerValue.history.historyType === HistoryWindowType.LAST_INTERVAL) { this.innerValue.displayValue += this.translate.instant('timewindow.last-prefix') + ' ' + this.millisecondsToTimeStringPipe.transform(this.innerValue.history.timewindowMs); diff --git a/ui-ngx/src/app/shared/models/widget.models.ts b/ui-ngx/src/app/shared/models/widget.models.ts index c654c87784..0472949680 100644 --- a/ui-ngx/src/app/shared/models/widget.models.ts +++ b/ui-ngx/src/app/shared/models/widget.models.ts @@ -367,6 +367,23 @@ export function datasourcesHasAggregation(datasources?: Array): bool return false; } +export function datasourcesHasOnlyComparisonAggregation(datasources?: Array): boolean { + if (!datasourcesHasAggregation(datasources)) { + return false; + } + if (datasources) { + const foundDatasource = datasources.find(datasource => { + const found = datasource.dataKeys && datasource.dataKeys.find(key => key.type === DataKeyType.timeseries && + key.aggregationType && key.aggregationType !== AggregationType.NONE && !key.comparisonEnabled); + return !!found; + }); + if (foundDatasource) { + return false; + } + } + return true; +} + export interface FormattedData { $datasource: Datasource; entityName: string; From 4bc66473db658e648695decfc8dae434583e6e2a Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 15 Sep 2022 17:26:28 +0300 Subject: [PATCH 16/18] UI: Improve latest data key aggregation config --- .../widget/data-key-config.component.html | 22 +++++++++------ .../widget/data-key-config.component.scss | 1 + ui-ngx/src/app/shared/models/widget.models.ts | 6 ++--- .../assets/locale/locale.constant-en_US.json | 27 ++++++++++--------- 4 files changed, 33 insertions(+), 23 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html index fe4df7ca62..9088f5cc33 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html @@ -63,24 +63,30 @@
- - datakey.aggregation-type + + datakey.aggregation - {{ (aggregation === aggregationTypes.NONE ? 'datakey.latest-value' : aggregationTypesTranslations.get(aggregationTypes[aggregation])) | translate }} + {{ aggregationTypesTranslations.get(aggregationTypes[aggregation]) | translate }} - {{ dataKeyFormGroup.get('aggregationType').value ? (dataKeyAggregationTypeHintTranslations.get(aggregationTypes[dataKeyFormGroup.get('aggregationType').value]) | translate) : '' }} + + {{ dataKeyFormGroup.get('aggregationType').value ? (dataKeyAggregationTypeHintTranslations.get(aggregationTypes[dataKeyFormGroup.get('aggregationType').value]) | translate) : '' }} +
+ {{ 'datakey.aggregation-type-hint-common' | translate }} +
+
- widgets.chart.comparison-settings + datakey.delta-calculation - + - {{ 'widgets.chart.enable-comparison' | translate }} + {{ 'datakey.enable-delta-calculation' | translate }} + {{ 'datakey.enable-delta-calculation-hint' | translate }} @@ -113,7 +119,7 @@ - datakey.comparison-result + datakey.delta-calculation-result {{ comparisonResultTypeTranslations.get(comparisonResultTypes[comparisonResultType]) | translate }} diff --git a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.scss b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.scss index d0d7e56137..7fd1fba872 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.scss +++ b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.scss @@ -83,6 +83,7 @@ .mat-expansion-panel-header { padding: 0; color: rgba(0, 0, 0, 0.87); + height: 140px; &:hover { background: none; diff --git a/ui-ngx/src/app/shared/models/widget.models.ts b/ui-ngx/src/app/shared/models/widget.models.ts index 0472949680..92b977e346 100644 --- a/ui-ngx/src/app/shared/models/widget.models.ts +++ b/ui-ngx/src/app/shared/models/widget.models.ts @@ -267,9 +267,9 @@ export enum ComparisonResultType { export const comparisonResultTypeTranslationMap = new Map( [ - [ComparisonResultType.PREVIOUS_VALUE, 'datakey.comparison-result-previous-value'], - [ComparisonResultType.DELTA_ABSOLUTE, 'datakey.comparison-result-delta-absolute'], - [ComparisonResultType.DELTA_PERCENT, 'datakey.comparison-result-delta-percent'] + [ComparisonResultType.PREVIOUS_VALUE, 'datakey.delta-calculation-result-previous-value'], + [ComparisonResultType.DELTA_ABSOLUTE, 'datakey.delta-calculation-result-delta-absolute'], + [ComparisonResultType.DELTA_PERCENT, 'datakey.delta-calculation-result-delta-percent'] ] ); diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index e64d7b9b78..1fd934d92d 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1040,18 +1040,21 @@ "prev-value-description": "result of the previous function call;", "time-prev-description": "timestamp of the previous value;", "prev-orig-value-description": "original previous value;", - "aggregation-type": "Aggregation type", - "latest-value": "Latest value", - "aggregation-type-none-hint": "Take latest value", - "aggregation-type-min-hint": "Take min value", - "aggregation-type-max-hint": "Take max value", - "aggregation-type-avg-hint": "Calculate average value", - "aggregation-type-sum-hint": "Calculate sum value", - "aggregation-type-count-hint": "Calculate count value", - "comparison-result": "Comparison result", - "comparison-result-previous-value": "Previous value", - "comparison-result-delta-absolute": "Delta (absolute)", - "comparison-result-delta-percent": "Delta (percent)" + "aggregation": "Aggregation", + "aggregation-type-hint-common": "For performance reasons, the aggregated values calculation is available only for fixed time intervals like \"current day\", \"current month\", etc, and is not available for sliding window intervals like 'last 30 minutes' or 'last 24 hours'.", + "aggregation-type-none-hint": "Take latest value.", + "aggregation-type-min-hint": "Find minimum value among data points within a selected time window.", + "aggregation-type-max-hint": "Find maximum value among data points within a selected time window.", + "aggregation-type-avg-hint": "Calculate an average value among data points within a selected time window.", + "aggregation-type-sum-hint": "Sum all values of the data points within a selected time window.", + "aggregation-type-count-hint": "Total number of the data points within a selected time window.", + "delta-calculation": "Delta calculation", + "enable-delta-calculation": "Enable delta calculation", + "enable-delta-calculation-hint": "When enabled, the data key value is calculated based on the aggregated values for a selected time window and a specified comparison period. For performance reasons, the delta calculation is available only for history time windows and not for real-time values. For example, you may calculate the delta between the energy consumption for yesterday compared to the energy consumption for the day before yesterday.", + "delta-calculation-result": "Delta calculation result", + "delta-calculation-result-previous-value": "Previous value", + "delta-calculation-result-delta-absolute": "Delta (absolute)", + "delta-calculation-result-delta-percent": "Delta (percent)" }, "datasource": { "type": "Datasource type", From 89c715cd189cb68675436cbde94bf113fa8f02bc Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 16 Sep 2022 11:55:48 +0300 Subject: [PATCH 17/18] Fix Cassandra DAO --- .../service/telemetry/cmd/v2/AggHistoryCmd.java | 1 - .../service/telemetry/cmd/v2/EntityDataCmd.java | 12 ------------ .../server/dao/timeseries/TimeseriesService.java | 1 - .../server/dao/model/ModelConstants.java | 8 ++++---- .../dao/timeseries/AggregatePartitionsFunction.java | 13 +++++++------ 5 files changed, 11 insertions(+), 24 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/AggHistoryCmd.java b/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/AggHistoryCmd.java index 4d191590c7..423c55bd0d 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/AggHistoryCmd.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/AggHistoryCmd.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.telemetry.cmd.v2; import lombok.Data; -import org.thingsboard.server.common.data.kv.Aggregation; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/EntityDataCmd.java b/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/EntityDataCmd.java index e18b1a745b..8a187afa7b 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/EntityDataCmd.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/EntityDataCmd.java @@ -62,16 +62,4 @@ public class EntityDataCmd extends DataCmd { return historyCmd != null || latestCmd != null || tsCmd != null || aggHistoryCmd != null || aggTsCmd != null; } - @JsonIgnore - public boolean hasRegularCmds() { - return historyCmd != null || latestCmd != null || tsCmd != null; - } - - @JsonIgnore - public boolean hasAggCmds() { - return aggHistoryCmd != null || aggTsCmd != null; - } - - - } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java index 0015ee73be..cf17eec88d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.Collection; import java.util.List; -import java.util.Map; /** * @author Andrew Shvayka diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 5218de60e4..e27bf174a9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -631,14 +631,14 @@ public class ModelConstants { protected static final String[] NONE_AGGREGATION_COLUMNS = new String[]{LONG_VALUE_COLUMN, DOUBLE_VALUE_COLUMN, BOOLEAN_VALUE_COLUMN, STRING_VALUE_COLUMN, JSON_VALUE_COLUMN, KEY_COLUMN, TS_COLUMN}; - protected static final String[] COUNT_AGGREGATION_COLUMNS = new String[]{count(LONG_VALUE_COLUMN), count(DOUBLE_VALUE_COLUMN), count(BOOLEAN_VALUE_COLUMN), count(STRING_VALUE_COLUMN), count(JSON_VALUE_COLUMN)}; + protected static final String[] COUNT_AGGREGATION_COLUMNS = new String[]{count(LONG_VALUE_COLUMN), count(DOUBLE_VALUE_COLUMN), count(BOOLEAN_VALUE_COLUMN), count(STRING_VALUE_COLUMN), count(JSON_VALUE_COLUMN), max(TS_COLUMN)}; protected static final String[] MIN_AGGREGATION_COLUMNS = - ArrayUtils.addAll(COUNT_AGGREGATION_COLUMNS, new String[]{min(LONG_VALUE_COLUMN), min(DOUBLE_VALUE_COLUMN), min(BOOLEAN_VALUE_COLUMN), min(STRING_VALUE_COLUMN), min(JSON_VALUE_COLUMN), max(TS_COLUMN)}); + ArrayUtils.addAll(COUNT_AGGREGATION_COLUMNS, new String[]{min(LONG_VALUE_COLUMN), min(DOUBLE_VALUE_COLUMN), min(BOOLEAN_VALUE_COLUMN), min(STRING_VALUE_COLUMN), min(JSON_VALUE_COLUMN)}); protected static final String[] MAX_AGGREGATION_COLUMNS = - ArrayUtils.addAll(COUNT_AGGREGATION_COLUMNS, new String[]{max(LONG_VALUE_COLUMN), max(DOUBLE_VALUE_COLUMN), max(BOOLEAN_VALUE_COLUMN), max(STRING_VALUE_COLUMN), max(JSON_VALUE_COLUMN), max(TS_COLUMN)}); + ArrayUtils.addAll(COUNT_AGGREGATION_COLUMNS, new String[]{max(LONG_VALUE_COLUMN), max(DOUBLE_VALUE_COLUMN), max(BOOLEAN_VALUE_COLUMN), max(STRING_VALUE_COLUMN), max(JSON_VALUE_COLUMN)}); protected static final String[] SUM_AGGREGATION_COLUMNS = - ArrayUtils.addAll(COUNT_AGGREGATION_COLUMNS, new String[]{sum(LONG_VALUE_COLUMN), sum(DOUBLE_VALUE_COLUMN), max(TS_COLUMN)}); + ArrayUtils.addAll(COUNT_AGGREGATION_COLUMNS, new String[]{sum(LONG_VALUE_COLUMN), sum(DOUBLE_VALUE_COLUMN)}); protected static final String[] AVG_AGGREGATION_COLUMNS = SUM_AGGREGATION_COLUMNS; public static String min(String s) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java index 7a19e9fbf8..e774ad489a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java @@ -49,12 +49,13 @@ public class AggregatePartitionsFunction implements com.google.common.util.concu private static final int BOOL_CNT_POS = 2; private static final int STR_CNT_POS = 3; private static final int JSON_CNT_POS = 4; - private static final int LONG_POS = 5; - private static final int DOUBLE_POS = 6; - private static final int BOOL_POS = 7; - private static final int STR_POS = 8; - private static final int JSON_POS = 9; - private static final int MAX_TS_POS = 10; + private static final int MAX_TS_POS = 5; + private static final int LONG_POS = 6; + private static final int DOUBLE_POS = 7; + private static final int BOOL_POS = 8; + private static final int STR_POS = 9; + private static final int JSON_POS = 10; + private final Aggregation aggregation; private final String key; From eebe8d8cbf07fd187ba17e5eb3ceea153f09b549 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 16 Sep 2022 15:20:27 +0300 Subject: [PATCH 18/18] Fix invalid logic related to endTs in read queries --- .../timescale/ts/TimescaleTsKvEntity.java | 1 - ...stractChunkedAggregationTimeseriesDao.java | 31 ++++++++++--------- .../timescale/AggregationRepository.java | 2 +- .../timescale/TimescaleTimeseriesDao.java | 15 +++++++-- .../CassandraBaseTimeseriesDao.java | 6 ++-- .../timeseries/BaseTimeseriesServiceTest.java | 18 +++++------ ...ctChunkedAggregationTimeseriesDaoTest.java | 23 +++++++------- 7 files changed, 53 insertions(+), 43 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/ts/TimescaleTsKvEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/ts/TimescaleTsKvEntity.java index 9e072e25e4..ba5ad92f4f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/ts/TimescaleTsKvEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/ts/TimescaleTsKvEntity.java @@ -79,7 +79,6 @@ import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.F @ColumnResult(name = "longValueCount", type = Long.class), @ColumnResult(name = "doubleValueCount", type = Long.class), @ColumnResult(name = "jsonValueCount", type = Long.class), - @ColumnResult(name = "jsonValueCount", type = Long.class), @ColumnResult(name = "maxAggTs", type = Long.class), } ) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 9acbfe85fb..22c4661fcd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -121,15 +121,14 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq return Futures.immediateFuture(findAllAsyncWithLimit(entityId, query)); } else { List>> futures = new ArrayList<>(); - long endPeriod = query.getEndTs(); long startPeriod = query.getStartTs(); + long endPeriod = Math.max(query.getStartTs() + 1, query.getEndTs()); long step = query.getInterval(); - while (startPeriod <= endPeriod) { + while (startPeriod < endPeriod) { long startTs = startPeriod; - long endTs = Math.min(startPeriod + step, endPeriod + 1); + long endTs = Math.min(startPeriod + step, endPeriod); long ts = startTs + (endTs - startTs) / 2; - ListenableFuture> aggregateTsKvEntry = - service.submit(() -> findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation())); + ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation()); futures.add(aggregateTsKvEntry); startPeriod = endTs; } @@ -152,16 +151,18 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq return new ReadTsKvQueryResult(query.getId(), tsKvEntries, lastTs); } - Optional findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { - TsKvEntity entity = switchAggregation(entityId, key, startTs, endTs, aggregation); - if (entity != null && entity.isNotEmpty()) { - entity.setEntityId(entityId.getId()); - entity.setStrKey(key); - entity.setTs(ts); - return Optional.of(entity); - } else { - return Optional.empty(); - } + ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { + return service.submit(() -> { + TsKvEntity entity = switchAggregation(entityId, key, startTs, endTs, aggregation); + if (entity != null && entity.isNotEmpty()) { + entity.setEntityId(entityId.getId()); + entity.setStrKey(key); + entity.setTs(ts); + return Optional.of(entity); + } else { + return Optional.empty(); + } + }); } protected TsKvEntity switchAggregation(EntityId entityId, String key, long startTs, long endTs, Aggregation aggregation) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java index db0a1753d8..d37234dfde 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java @@ -39,7 +39,7 @@ public class AggregationRepository { public static final String FROM_WHERE_CLAUSE = "FROM ts_kv tskv WHERE " + "tskv.entity_id = cast(:entityId AS uuid) " + "AND tskv.key= cast(:entityKey AS int) " + - "AND tskv.ts > :startTs AND tskv.ts <= :endTs " + + "AND tskv.ts >= :startTs AND tskv.ts < :endTs " + "GROUP BY tskv.entity_id, tskv.key, tsBucket " + "ORDER BY tskv.entity_id, tskv.key, tsBucket"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 76dabb24f6..6e9f9e61b8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -155,7 +155,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements return Futures.immediateFuture(findAllAsyncWithLimit(entityId, query)); } else { long startTs = query.getStartTs(); - long endTs = query.getEndTs(); + long endTs = Math.max(query.getStartTs() + 1, query.getEndTs()); long timeBucket = query.getInterval(); List> data = findAllAndAggregateAsync(entityId, query.getKey(), startTs, endTs, timeBucket, query.getAggregation()); return getReadTsKvQueryResultFuture(query, Futures.immediateFuture(data)); @@ -185,7 +185,18 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements } private List> findAllAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long timeBucket, Aggregation aggregation) { - List timescaleTsKvEntities = switchAggregation(key, startTs, endTs, timeBucket, aggregation, entityId.getId()); + long interval = endTs - startTs; + long remainingPart = interval % timeBucket; + List timescaleTsKvEntities; + if (remainingPart == 0) { + timescaleTsKvEntities = switchAggregation(key, startTs, endTs, timeBucket, aggregation, entityId.getId()); + } else { + interval = interval - remainingPart; + timescaleTsKvEntities = new ArrayList<>(); + timescaleTsKvEntities.addAll(switchAggregation(key, startTs, startTs + interval, timeBucket, aggregation, entityId.getId())); + timescaleTsKvEntities.addAll(switchAggregation(key, startTs + interval, endTs, remainingPart, aggregation, entityId.getId())); + } + if (!CollectionUtils.isEmpty(timescaleTsKvEntities)) { List> result = new ArrayList<>(); timescaleTsKvEntities.forEach(entity -> { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 35eb5b8a18..367ad942a7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -268,12 +268,12 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD return findAllAsyncWithLimit(tenantId, entityId, query); } else { long startPeriod = query.getStartTs(); - long endPeriod = query.getEndTs(); + long endPeriod = Math.max(query.getStartTs() + 1, query.getEndTs()); long step = Math.max(query.getInterval(), MIN_AGGREGATION_STEP_MS); List>> futures = new ArrayList<>(); - while (startPeriod <= endPeriod) { + while (startPeriod < endPeriod) { long startTs = startPeriod; - long endTs = Math.min(startPeriod + step, endPeriod + 1); + long endTs = Math.min(startPeriod + step, endPeriod); long ts = endTs - startTs; ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); futures.add(findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs))); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 5339863044..4e2fd17846 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -213,17 +213,17 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { } saveEntries(deviceId, TS + 100L + 1L); - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 100, 101, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 100, 100, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(1, entries.size()); - Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 10L)), entries.get(0)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(1, entries.size()); - Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 10L)), entries.get(0)); } @Test @@ -240,14 +240,14 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 75000 - 1, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 75000 - 1, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); } @Test @@ -264,14 +264,14 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); } @Test @@ -286,14 +286,14 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 75000 - 1, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 75000 - 1, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); } @Test diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 40705809fd..9c4b1cbbea 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -19,6 +19,8 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; @@ -44,13 +46,13 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { final int LIMIT = 1; final String TEMP = "temp"; final String DESC = "DESC"; - AbstractChunkedAggregationTimeseriesDao tsDao; + private AbstractChunkedAggregationTimeseriesDao tsDao; @Before public void setUp() throws Exception { tsDao = spy(AbstractChunkedAggregationTimeseriesDao.class); Optional optionalListenableFuture = Optional.of(mock(TsKvEntry.class)); - willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + willReturn(Futures.immediateFuture(optionalListenableFuture)).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); willReturn(Futures.immediateFuture(mock(ReadTsKvQueryResult.class))).given(tsDao).getReadTsKvQueryResultFuture(any(), any()); } @@ -58,11 +60,11 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenLastIntervalShorterThanOthersAndEqualsEndTs() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2000, LIMIT, COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2001, 1001, LIMIT, COUNT, DESC); - ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3001, 2501, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3000, 2501, LIMIT, COUNT, DESC); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 2001, getTsForReadTsKvQuery(1, 2001), COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 2001, 3000 + 1, getTsForReadTsKvQuery(2001, 3001), COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 2001, 3000, getTsForReadTsKvQuery(2001, 3000), COUNT); } @Test @@ -72,19 +74,17 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); assertThat(tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query)).isNotNull(); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000 + 1, getTsForReadTsKvQuery(1, 3001), COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000, getTsForReadTsKvQuery(1, 3000), COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodMinusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2999, LIMIT, COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1500, LIMIT, COUNT, DESC); - ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 3000, 3001, 3000, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000, getTsForReadTsKvQuery(1, 3000), COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 3000, 3001, getTsForReadTsKvQuery(3000, 3001), COUNT); } @@ -95,7 +95,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3001, getTsForReadTsKvQuery(1, 3001), COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000, getTsForReadTsKvQuery(1, 3000), COUNT); } @Test @@ -135,7 +135,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3001, getTsForReadTsKvQuery(1, 3001), COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000, getTsForReadTsKvQuery(1, 3000), COUNT); } @Test @@ -145,8 +145,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1000)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); for (long i = 1; i <= 3000; i += 3) { - ReadTsKvQuery querySub = new BaseReadTsKvQuery(TEMP, i, i + 3, i + (i + 3 - i) / 2, LIMIT, COUNT, DESC); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, querySub.getKey(), i, i + 3, getTsForReadTsKvQuery(i, i + 3), COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, TEMP, i, Math.min(i + 3, 3000), getTsForReadTsKvQuery(i, i + 3), COUNT); } }