Merge Alarms fetch improvements

application/src/main/java/org/thingsboard/server/controller/AlarmController.java

@@ -44,6 +44,8 @@ import org.thingsboard.server.common.data.page.TimePageLink;
 import org.thingsboard.server.service.security.permission.Operation;
 import org.thingsboard.server.service.security.permission.Resource;
 
+import java.util.UUID;
+
 @RestController
 @RequestMapping("/api")
 public class AlarmController extends BaseController {
@@ -155,6 +157,7 @@ public class AlarmController extends BaseController {
             @RequestParam(required = false) String sortOrder,
             @RequestParam(required = false) Long startTime,
             @RequestParam(required = false) Long endTime,
+            @RequestParam(required = false) String offset,
             @RequestParam(required = false) Boolean fetchOriginator
     ) throws ThingsboardException {
         checkParameter("EntityId", strEntityId);
@@ -168,8 +171,12 @@ public class AlarmController extends BaseController {
         }
         checkEntityId(entityId, Operation.READ);
         TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime);
+        UUID idOffsetUuid = null;
+        if (StringUtils.isNotEmpty(offset)) {
+            idOffsetUuid = toUUID(offset);
+        }
         try {
-            return checkNotNull(alarmService.findAlarms(getCurrentUser().getTenantId(), new AlarmQuery(entityId, pageLink, alarmSearchStatus, alarmStatus, fetchOriginator)).get());
+            return checkNotNull(alarmService.findAlarms(getCurrentUser().getTenantId(), new AlarmQuery(entityId, pageLink, alarmSearchStatus, alarmStatus, fetchOriginator, idOffsetUuid)).get());
         } catch (Exception e) {
             throw handleException(e);
         }

common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmQuery.java

@@ -22,6 +22,8 @@ import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.page.TimePageLink;
 
+import java.util.UUID;
+
 /**
  * Created by ashvayka on 11.05.17.
  */
@@ -35,5 +37,6 @@ public class AlarmQuery {
     private AlarmSearchStatus searchStatus;
     private AlarmStatus status;
     private Boolean fetchOriginator;
+    private UUID idOffset;
 
 }

dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java

@@ -300,7 +300,7 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ
         AlarmSeverity highestSeverity = null;
         AlarmQuery query;
         while (hasNext && AlarmSeverity.CRITICAL != highestSeverity) {
-            query = new AlarmQuery(entityId, nextPageLink, alarmSearchStatus, alarmStatus, false);
+            query = new AlarmQuery(entityId, nextPageLink, alarmSearchStatus, alarmStatus, false, null);
             PageData<AlarmInfo> alarms = alarmDao.findAlarms(tenantId, query);
             if (alarms.hasNext()) {
                 nextPageLink = nextPageLink.nextPageLink();

dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java

@@ -33,7 +33,7 @@ import java.util.List;
 @SqlDao
 public interface AlarmRepository extends CrudRepository<AlarmEntity, String> {
 
-    @Query("SELECT a FROM AlarmEntity a WHERE a.originatorId = :originatorId AND a.type = :alarmType ORDER BY startTs DESC")
+    @Query("SELECT a FROM AlarmEntity a WHERE a.originatorId = :originatorId AND a.type = :alarmType ORDER BY a.startTs DESC")
     List<AlarmEntity> findLatestByOriginatorAndType(@Param("originatorId") String originatorId,
                                                     @Param("alarmType") String alarmType,
                                                     Pageable pageable);
@@ -48,6 +48,7 @@ public interface AlarmRepository extends CrudRepository<AlarmEntity, String> {
             "AND re.fromType = :affectedEntityType " +
             "AND (:startId IS NULL OR a.id >= :startId) " +
             "AND (:endId IS NULL OR a.id <= :endId) " +
+            "AND (:idOffset IS NULL OR a.id < :idOffset) " +
             "AND (LOWER(a.type) LIKE LOWER(CONCAT(:searchText, '%'))" +
             "OR LOWER(a.severity) LIKE LOWER(CONCAT(:searchText, '%'))" +
             "OR LOWER(a.status) LIKE LOWER(CONCAT(:searchText, '%')))")
@@ -57,6 +58,7 @@ public interface AlarmRepository extends CrudRepository<AlarmEntity, String> {
                                      @Param("relationType") String relationType,
                                      @Param("startId") String startId,
                                      @Param("endId") String endId,
+                                     @Param("idOffset") String idOffset,
                                      @Param("searchText") String searchText,
                                      Pageable pageable);
 }

dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java

@@ -117,6 +117,7 @@ public class JpaAlarmDao extends JpaAbstractDao<AlarmEntity, Alarm> implements A
                     relationType,
                     startTimeToId(query.getPageLink().getStartTime()),
                     endTimeToId(query.getPageLink().getEndTime()),
+                    query.getIdOffset() != null ? UUIDConverter.fromTimeUUID(query.getIdOffset()) : null,
                     Objects.toString(query.getPageLink().getTextSearch(), ""),
                     DaoUtil.toPageable(query.getPageLink())
             )

ui-ngx/src/app/core/api/widget-api.models.ts

@@ -190,6 +190,8 @@ export interface WidgetSubscriptionOptions {
   alarmSource?: Datasource;
   alarmSearchStatus?: AlarmSearchStatus;
   alarmsPollingInterval?: number;
+  alarmsMaxCountLoad?: number;
+  alarmsFetchSize?: number;
   datasources?: Array<Datasource>;
   targetDeviceAliasIds?: Array<string>;
   targetDeviceIds?: Array<string>;

ui-ngx/src/app/core/api/widget-subscription.ts

@@ -93,6 +93,8 @@ export class WidgetSubscription implements IWidgetSubscription {
   }
 
   alarmsPollingInterval: number;
+  alarmsMaxCountLoad: number;
+  alarmsFetchSize: number;
   alarmSourceListener: AlarmSourceListener;
 
   loadingData: boolean;
@@ -153,6 +155,10 @@ export class WidgetSubscription implements IWidgetSubscription {
         options.alarmSearchStatus : AlarmSearchStatus.ANY;
       this.alarmsPollingInterval = isDefined(options.alarmsPollingInterval) ?
         options.alarmsPollingInterval : 5000;
+      this.alarmsMaxCountLoad = isDefined(options.alarmsMaxCountLoad) ?
+        options.alarmsMaxCountLoad : 0;
+      this.alarmsFetchSize = isDefined(options.alarmsFetchSize) ?
+        options.alarmsFetchSize : 100;
       this.alarmSourceListener = null;
       this.alarms = [];
       this.originalTimewindow = null;
@@ -712,6 +718,8 @@ export class WidgetSubscription implements IWidgetSubscription {
       alarmSource: this.alarmSource,
       alarmSearchStatus: this.alarmSearchStatus,
       alarmsPollingInterval: this.alarmsPollingInterval,
+      alarmsMaxCountLoad: this.alarmsMaxCountLoad,
+      alarmsFetchSize: this.alarmsFetchSize,
       alarmsUpdated: alarms => this.alarmsUpdated(alarms)
     };
     this.alarms = null;

ui-ngx/src/app/core/http/alarm.service.ts

@@ -38,12 +38,15 @@ import { Direction, SortOrder } from '@shared/models/page/sort-order';
 import { concatMap, expand, map, toArray } from 'rxjs/operators';
 import { EMPTY } from 'rxjs';
 import Timeout = NodeJS.Timeout;
+import { isDefined } from '@core/utils';
 
 interface AlarmSourceListenerQuery {
   entityType: EntityType;
   entityId: string;
   alarmSearchStatus: AlarmSearchStatus;
   alarmStatus: AlarmStatus;
+  alarmsMaxCountLoad: number;
+  alarmsFetchSize: number;
   fetchOriginator?: boolean;
   limit?: number;
   interval?: number;
@@ -58,6 +61,8 @@ export interface AlarmSourceListener {
   alarmSource: Datasource;
   alarmsPollingInterval: number;
   alarmSearchStatus: AlarmSearchStatus;
+  alarmsMaxCountLoad: number;
+  alarmsFetchSize: number;
   alarmsUpdated: (alarms: Array<AlarmInfo>) => void;
   lastUpdateTs?: number;
   alarmsQuery?: AlarmSourceListenerQuery;
@@ -132,7 +137,9 @@ export class AlarmService {
         entityType: alarmSource.entityType,
         entityId: alarmSource.entityId,
         alarmSearchStatus: alarmSourceListener.alarmSearchStatus,
-        alarmStatus: null
+        alarmStatus: null,
+        alarmsMaxCountLoad: alarmSourceListener.alarmsMaxCountLoad,
+        alarmsFetchSize: alarmSourceListener.alarmsFetchSize
       };
       const originatorKeys = alarmSource.dataKeys.filter(dataKey => dataKey.name.toLocaleLowerCase().includes('originator'));
       if (originatorKeys.length) {
@@ -186,32 +193,49 @@ export class AlarmService {
         null,
         sortOrder);
     } else if (alarmsQuery.interval) {
-      pageLink = new TimePageLink(100, 0,
+      pageLink = new TimePageLink(alarmsQuery.alarmsFetchSize || 100, 0,
         null,
         sortOrder, time - alarmsQuery.interval);
     } else if (alarmsQuery.startTime) {
-      pageLink = new TimePageLink(100, 0,
+      pageLink = new TimePageLink(alarmsQuery.alarmsFetchSize || 100, 0,
         null,
         sortOrder, Math.round(alarmsQuery.startTime));
       if (alarmsQuery.endTime) {
         pageLink.endTime = Math.round(alarmsQuery.endTime);
       }
     }
-    return this.fetchAlarms(alarmsQuery, pageLink);
+    let leftToLoad;
+    if (isDefined(alarmsQuery.alarmsMaxCountLoad) && alarmsQuery.alarmsMaxCountLoad !== 0) {
+      leftToLoad = alarmsQuery.alarmsMaxCountLoad;
+      if (leftToLoad < pageLink.pageSize) {
+        pageLink.pageSize = leftToLoad;
+      }
+    }
+    return this.fetchAlarms(alarmsQuery, pageLink, leftToLoad);
   }
 
   private fetchAlarms(query: AlarmSourceListenerQuery,
-                      pageLink: TimePageLink): Observable<Array<AlarmInfo>> {
+                      pageLink: TimePageLink, leftToLoad?: number): Observable<Array<AlarmInfo>> {
     const alarmQuery = new AlarmQuery(
       {id: query.entityId, entityType: query.entityType},
       pageLink,
       query.alarmSearchStatus,
       query.alarmStatus,
-      query.fetchOriginator);
+      query.fetchOriginator,
+      null);
     return this.getAlarms(alarmQuery, {ignoreLoading: true}).pipe(
       expand((data) => {
-        if (data.hasNext && !query.limit) {
-          alarmQuery.pageLink.page += 1;
+        let continueLoad = data.hasNext && !query.limit;
+        if (continueLoad && isDefined(leftToLoad)) {
+          leftToLoad -= data.data.length;
+          if (leftToLoad === 0) {
+            continueLoad = false;
+          } else if (leftToLoad < alarmQuery.pageLink.pageSize) {
+            alarmQuery.pageLink.pageSize = leftToLoad;
+          }
+        }
+        if (continueLoad) {
+          alarmQuery.offset = data.data[data.data.length-1].id.id;
           return this.getAlarms(alarmQuery, {ignoreLoading: true});
         } else {
           return EMPTY;

ui-ngx/src/app/modules/home/components/alarm/alarm-table-config.ts

@@ -110,7 +110,7 @@ export class AlarmTableConfig extends EntityTableConfig<AlarmInfo, TimePageLink>
   }
 
   fetchAlarms(pageLink: TimePageLink): Observable<PageData<AlarmInfo>> {
-    const query = new AlarmQuery(this.entityId, pageLink, this.searchStatus, null, true);
+    const query = new AlarmQuery(this.entityId, pageLink, this.searchStatus, null, true, null);
     return this.alarmService.getAlarms(query);
   }
 

ui-ngx/src/app/modules/home/components/widget/widget-config.component.html

@@ -36,29 +36,60 @@
                          fxFlex formControlName="timewindow"></tb-timewindow>
         </section>
       </div>
-      <div *ngIf="widgetType === widgetTypes.alarm" fxLayout="column" fxLayoutGap="8px" fxLayoutAlign="center"
-           fxLayout.gt-sm="row" fxLayoutAlign.gt-sm="start center">
-        <mat-form-field fxFlex class="mat-block">
-          <mat-label translate>alarm.alarm-status</mat-label>
-          <mat-select matInput formControlName="alarmSearchStatus">
-            <mat-option *ngFor="let searchStatus of alarmSearchStatuses" [value]="searchStatus">
-              {{ ('alarm.search-status.' + searchStatus) | translate }}
-            </mat-option>
-          </mat-select>
-        </mat-form-field>
-        <mat-form-field fxFlex class="mat-block">
-          <mat-label translate>alarm.polling-interval</mat-label>
-          <input matInput required
-                 formControlName="alarmsPollingInterval"
-                 type="number"
-                 step="1"/>
-          <mat-error *ngIf="dataSettings.get('alarmsPollingInterval').hasError('required')">
-            {{ 'alarm.polling-interval-required' | translate }}
-          </mat-error>
-          <mat-error *ngIf="dataSettings.get('alarmsPollingInterval').hasError('min')">
-            {{ 'alarm.min-polling-interval-message' | translate }}
-          </mat-error>
-        </mat-form-field>
+      <div *ngIf="widgetType === widgetTypes.alarm" fxLayout="column" fxLayoutAlign="center">
+        <div fxLayout.gt-sm="row" fxLayoutAlign.gt-sm="start center" fxLayoutGap="8px">
+          <mat-form-field fxFlex class="mat-block">
+            <mat-label translate>alarm.alarm-status</mat-label>
+            <mat-select matInput formControlName="alarmSearchStatus">
+              <mat-option *ngFor="let searchStatus of alarmSearchStatuses" [value]="searchStatus">
+                {{ ('alarm.search-status.' + searchStatus) | translate }}
+              </mat-option>
+            </mat-select>
+          </mat-form-field>
+          <mat-form-field fxFlex class="mat-block">
+            <mat-label translate>alarm.polling-interval</mat-label>
+            <input matInput required
+                   formControlName="alarmsPollingInterval"
+                   type="number"
+                   step="1"/>
+            <mat-error *ngIf="dataSettings.get('alarmsPollingInterval').hasError('required')">
+              {{ 'alarm.polling-interval-required' | translate }}
+            </mat-error>
+            <mat-error *ngIf="dataSettings.get('alarmsPollingInterval').hasError('min')">
+              {{ 'alarm.min-polling-interval-message' | translate }}
+            </mat-error>
+          </mat-form-field>
+        </div>
+        <div fxLayout.gt-sm="row" fxLayoutAlign.gt-sm="start center" fxLayoutGap="8px">
+          <mat-form-field fxFlex class="mat-block">
+            <mat-label translate>alarm.max-count-load</mat-label>
+            <input matInput required
+                   formControlName="alarmsMaxCountLoad"
+                   type="number"
+                   min="0"
+                   step="1">
+            <mat-error *ngIf="dataSettings.get('alarmsMaxCountLoad').hasError('required')">
+              {{ 'alarm.max-count-load-required' | translate }}
+            </mat-error>
+            <mat-error *ngIf="dataSettings.get('alarmsMaxCountLoad').hasError('min')">
+              {{ 'alarm.max-count-load-error-min' | translate }}
+            </mat-error>
+          </mat-form-field>
+          <mat-form-field fxFlex class="mat-block">
+            <mat-label translate>alarm.fetch-size</mat-label>
+            <input matInput required
+                   formControlName="alarmsFetchSize"
+                   type="number"
+                   min="10"
+                   step="1">
+            <mat-error *ngIf="dataSettings.get('alarmsFetchSize').hasError('required')">
+              {{ 'alarm.fetch-size-required' | translate }}
+            </mat-error>
+            <mat-error *ngIf="dataSettings.get('alarmsFetchSize').hasError('min')">
+              {{ 'alarm.fetch-size-error-min' | translate }}
+            </mat-error>
+          </mat-form-field>
+        </div>
       </div>
       <mat-expansion-panel class="tb-datasources" *ngIf="widgetType !== widgetTypes.rpc &&
                                   widgetType !== widgetTypes.alarm &&

ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts

@@ -287,6 +287,10 @@ export class WidgetConfigComponent extends PageComponent implements OnInit, Cont
         this.dataSettings.addControl('alarmSearchStatus', this.fb.control(null));
         this.dataSettings.addControl('alarmsPollingInterval', this.fb.control(null,
           [Validators.required, Validators.min(1)]));
+        this.dataSettings.addControl('alarmsMaxCountLoad', this.fb.control(null,
+          [Validators.required, Validators.min(0)]));
+        this.dataSettings.addControl('alarmsFetchSize', this.fb.control(null,
+          [Validators.required, Validators.min(10)]));
       }
     }
     if (this.modelValue.isDataEnabled) {
@@ -439,6 +443,14 @@ export class WidgetConfigComponent extends PageComponent implements OnInit, Cont
               { alarmsPollingInterval: isDefined(config.alarmsPollingInterval) ?
                   config.alarmsPollingInterval : 5}, {emitEvent: false}
             );
+            this.dataSettings.patchValue(
+              { alarmsMaxCountLoad: isDefined(config.alarmsMaxCountLoad) ?
+                  config.alarmsMaxCountLoad : 0}, {emitEvent: false}
+            );
+            this.dataSettings.patchValue(
+              { alarmsFetchSize: isDefined(config.alarmsFetchSize) ?
+                  config.alarmsFetchSize : 100}, {emitEvent: false}
+            );
             this.alarmSourceSettings.patchValue(
               config.alarmSource, {emitEvent: false}
             );

ui-ngx/src/app/modules/home/components/widget/widget.component.ts

@@ -828,6 +828,10 @@ export class WidgetComponent extends PageComponent implements OnInit, AfterViewI
           this.widget.config.alarmSearchStatus : AlarmSearchStatus.ANY;
         options.alarmsPollingInterval = isDefined(this.widget.config.alarmsPollingInterval) ?
           this.widget.config.alarmsPollingInterval * 1000 : 5000;
+        options.alarmsMaxCountLoad = isDefined(this.widget.config.alarmsMaxCountLoad) ?
+          this.widget.config.alarmsMaxCountLoad : 0;
+        options.alarmsFetchSize = isDefined(this.widget.config.alarmsFetchSize) ?
+          this.widget.config.alarmsFetchSize : 100;
       } else {
         options.datasources = deepClone(this.widget.config.datasources);
       }

ui-ngx/src/app/shared/models/alarm.models.ts

@@ -14,16 +14,14 @@
 /// limitations under the License.
 ///
 
-import {BaseData} from '@shared/models/base-data';
-import {AssetId} from '@shared/models/id/asset-id';
-import {TenantId} from '@shared/models/id/tenant-id';
-import {CustomerId} from '@shared/models/id/customer-id';
-import {AlarmId} from '@shared/models/id/alarm-id';
-import {EntityId} from '@shared/models/id/entity-id';
-import { ActionStatus } from '@shared/models/audit-log.models';
+import { BaseData } from '@shared/models/base-data';
+import { TenantId } from '@shared/models/id/tenant-id';
+import { AlarmId } from '@shared/models/id/alarm-id';
+import { EntityId } from '@shared/models/id/entity-id';
 import { TimePageLink } from '@shared/models/page/page-link';
 import { NULL_UUID } from '@shared/models/id/has-uuid';
 import { EntityType } from '@shared/models/entity-type.models';
+import { isString } from '@core/utils';
 
 export enum AlarmSeverity {
   CRITICAL = 'CRITICAL',
@@ -199,15 +197,17 @@ export class AlarmQuery {
   searchStatus: AlarmSearchStatus;
   status: AlarmStatus;
   fetchOriginator: boolean;
+  offset: string;
 
   constructor(entityId: EntityId, pageLink: TimePageLink,
               searchStatus: AlarmSearchStatus, status: AlarmStatus,
-              fetchOriginator: boolean) {
+              fetchOriginator: boolean, offset: string) {
     this.affectedEntityId = entityId;
     this.pageLink = pageLink;
     this.searchStatus = searchStatus;
     this.status = status;
     this.fetchOriginator = fetchOriginator;
+    this.offset = offset;
   }
 
   public toQuery(): string {
@@ -221,6 +221,9 @@ export class AlarmQuery {
     if (typeof this.fetchOriginator !== 'undefined' && this.fetchOriginator !== null) {
       query += `&fetchOriginator=${this.fetchOriginator}`;
     }
+    if (isString(this.offset) && this.offset.length) {
+      query += `&offset=${this.offset}`;
+    }
     return query;
   }
 

ui-ngx/src/app/shared/models/widget.models.ts

@@ -360,6 +360,8 @@ export interface WidgetConfig {
   alarmSource?: Datasource;
   alarmSearchStatus?: AlarmSearchStatus;
   alarmsPollingInterval?: number;
+  alarmsMaxCountLoad?: number;
+  alarmsFetchSize?: number;
   datasources?: Array<Datasource>;
   targetDeviceAliasIds?: Array<string>;
   [key: string]: any;