diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java index adb816a0ba..b583446f17 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java @@ -22,7 +22,6 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.Nullable; -import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; @@ -30,13 +29,13 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.springframework.web.socket.CloseStatus; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmDataQuery; +import org.thingsboard.server.common.data.query.ComparisonTsValue; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityKey; @@ -51,6 +50,9 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; import org.thingsboard.server.service.telemetry.TelemetryWebSocketSessionRef; +import org.thingsboard.server.service.telemetry.cmd.v2.AggHistoryCmd; +import org.thingsboard.server.service.telemetry.cmd.v2.AggKey; +import org.thingsboard.server.service.telemetry.cmd.v2.AggTimeSeriesCmd; import org.thingsboard.server.service.telemetry.cmd.v2.AlarmDataCmd; import org.thingsboard.server.service.telemetry.cmd.v2.AlarmDataUpdate; import org.thingsboard.server.service.telemetry.cmd.v2.EntityCountCmd; @@ -68,12 +70,12 @@ import javax.annotation.PreDestroy; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -83,6 +85,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +@SuppressWarnings("UnstableApiUsage") @Slf4j @TbCoreComponent @Service @@ -131,6 +134,8 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc private int maxEntitiesPerAlarmSubscription; @Value("${server.ws.dynamic_page_link.max_alarm_queries_per_refresh_interval:10}") private int maxAlarmQueriesPerRefreshInterval; + @Value("${ui.dashboard.max_datapoints_limit:50000}") + private int maxDatapointLimit; private ExecutorService wsCallBackExecutor; private boolean tsInSqlDB; @@ -165,7 +170,7 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc TbEntityDataSubCtx ctx = getSubCtx(session.getSessionId(), cmd.getCmdId()); if (ctx != null) { log.debug("[{}][{}] Updating existing subscriptions using: {}", session.getSessionId(), cmd.getCmdId(), cmd); - if (cmd.getLatestCmd() != null || cmd.getTsCmd() != null || cmd.getHistoryCmd() != null) { + if (cmd.hasAnyCmd()) { ctx.clearEntitySubscriptions(); } } else { @@ -173,6 +178,8 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc ctx = createSubCtx(session, cmd); } ctx.setCurrentCmd(cmd); + + // Fetch entity list using entity data query if (cmd.getQuery() != null) { if (ctx.getQuery() == null) { log.debug("[{}][{}] Initializing data using query: {}", session.getSessionId(), cmd.getCmdId(), cmd.getQuery()); @@ -204,43 +211,143 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc finalCtx.setRefreshTask(task); } } - ListenableFuture historyFuture; - if (cmd.getHistoryCmd() != null) { - log.trace("[{}][{}] Going to process history command: {}", session.getSessionId(), cmd.getCmdId(), cmd.getHistoryCmd()); - try { - historyFuture = handleHistoryCmd(ctx, cmd.getHistoryCmd()); - } catch (RuntimeException e) { - handleWsCmdRuntimeException(ctx.getSessionId(), e, cmd); - return; + + try { + List> cmdFutures = new ArrayList<>(); + if (cmd.getAggHistoryCmd() != null) { + cmdFutures.add(handleAggHistoryCmd(ctx, cmd.getAggHistoryCmd())); } - } else { - historyFuture = Futures.immediateFuture(ctx); + if (cmd.getAggTsCmd() != null) { + cmdFutures.add(handleAggTsCmd(ctx, cmd.getAggTsCmd())); + } + if (cmd.getHistoryCmd() != null) { + cmdFutures.add(handleHistoryCmd(ctx, cmd.getHistoryCmd())); + } + if (cmdFutures.isEmpty()) { + handleRegularCommands(ctx, cmd); + } else { + TbEntityDataSubCtx finalCtx = ctx; + Futures.addCallback(Futures.allAsList(cmdFutures), new FutureCallback<>() { + @Override + public void onSuccess(@Nullable List result) { + handleRegularCommands(finalCtx, cmd); + } + + @Override + public void onFailure(Throwable t) { + log.warn("[{}][{}] Failed to process command", finalCtx.getSessionId(), finalCtx.getCmdId()); + } + }, wsCallBackExecutor); + } + } catch (RuntimeException e) { + handleWsCmdRuntimeException(ctx.getSessionId(), e, cmd); + } + } + + private void handleRegularCommands(TbEntityDataSubCtx ctx, EntityDataCmd cmd) { + try { + if (cmd.getLatestCmd() != null || cmd.getTsCmd() != null) { + if (cmd.getLatestCmd() != null) { + handleLatestCmd(ctx, cmd.getLatestCmd()); + } + if (cmd.getTsCmd() != null) { + handleTimeSeriesCmd(ctx, cmd.getTsCmd()); + } + } else { + checkAndSendInitialData(ctx); + } + } catch (RuntimeException e) { + handleWsCmdRuntimeException(ctx.getSessionId(), e, cmd); } - Futures.addCallback(historyFuture, new FutureCallback<>() { - @Override - public void onSuccess(@Nullable TbEntityDataSubCtx theCtx) { + } + + private void checkAndSendInitialData(@Nullable TbEntityDataSubCtx theCtx) { + if (!theCtx.isInitialDataSent()) { + EntityDataUpdate update = new EntityDataUpdate(theCtx.getCmdId(), theCtx.getData(), null, theCtx.getMaxEntitiesPerDataSubscription()); + theCtx.sendWsMsg(update); + theCtx.setInitialDataSent(true); + } + } + + private ListenableFuture handleAggHistoryCmd(TbEntityDataSubCtx ctx, AggHistoryCmd cmd) { + ConcurrentMap queries = new ConcurrentHashMap<>(); + for (AggKey key : cmd.getKeys()) { + if (key.getPreviousValueOnly() == null || !key.getPreviousValueOnly()) { + var query = new BaseReadTsKvQuery(key.getKey(), cmd.getStartTs(), cmd.getEndTs(), cmd.getEndTs() - cmd.getStartTs(), 1, key.getAgg()); + queries.put(query.getId(), new ReadTsKvQueryInfo(key, query, false)); + } + if (key.getPreviousStartTs() != null && key.getPreviousEndTs() != null && key.getPreviousEndTs() >= key.getPreviousStartTs()) { + var query = new BaseReadTsKvQuery(key.getKey(), key.getPreviousStartTs(), key.getPreviousEndTs(), key.getPreviousEndTs() - key.getPreviousStartTs(), 1, key.getAgg()); + queries.put(query.getId(), new ReadTsKvQueryInfo(key, query, true)); + } + } + return handleAggCmd(ctx, cmd.getKeys(), queries, cmd.getStartTs(), cmd.getEndTs(), false); + } + + private ListenableFuture handleAggTsCmd(TbEntityDataSubCtx ctx, AggTimeSeriesCmd cmd) { + ConcurrentMap queries = new ConcurrentHashMap<>(); + for (AggKey key : cmd.getKeys()) { + var query = new BaseReadTsKvQuery(key.getKey(), cmd.getStartTs(), cmd.getStartTs() + cmd.getTimeWindow(), cmd.getTimeWindow(), 1, key.getAgg()); + queries.put(query.getId(), new ReadTsKvQueryInfo(key, query, false)); + } + return handleAggCmd(ctx, cmd.getKeys(), queries, cmd.getStartTs(), cmd.getStartTs() + cmd.getTimeWindow(), true); + } + + private ListenableFuture handleAggCmd(TbEntityDataSubCtx ctx, List keys, ConcurrentMap queries, + long startTs, long endTs, boolean subscribe) { + Map>> fetchResultMap = new HashMap<>(); + List entityDataList = ctx.getData().getData(); + List queryList = queries.values().stream().map(ReadTsKvQueryInfo::getQuery).collect(Collectors.toList()); + entityDataList.forEach(entityData -> fetchResultMap.put(entityData, + tsService.findAllByQueries(ctx.getTenantId(), entityData.getEntityId(), queryList))); + return Futures.transform(Futures.allAsList(fetchResultMap.values()), f -> { + // Map that holds last ts for each key for each entity. + Map> lastTsEntityMap = new HashMap<>(); + fetchResultMap.forEach((entityData, future) -> { try { - if (cmd.getLatestCmd() != null || cmd.getTsCmd() != null) { - if (cmd.getLatestCmd() != null) { - handleLatestCmd(theCtx, cmd.getLatestCmd()); - } - if (cmd.getTsCmd() != null) { - handleTimeSeriesCmd(theCtx, cmd.getTsCmd()); + Map lastTsMap = new HashMap<>(); + lastTsEntityMap.put(entityData, lastTsMap); + + List queryResults = future.get(); + if (queryResults != null) { + for (ReadTsKvQueryResult queryResult : queryResults) { + ReadTsKvQueryInfo queryInfo = queries.get(queryResult.getQueryId()); + ComparisonTsValue comparisonTsValue = entityData.getAggLatest().computeIfAbsent(queryInfo.getKey().getId(), agg -> new ComparisonTsValue()); + if (queryInfo.isPrevious()) { + comparisonTsValue.setPrevious(queryResult.toTsValue()); + } else { + comparisonTsValue.setCurrent(queryResult.toTsValue()); + lastTsMap.put(queryInfo.getQuery().getKey(), queryResult.getLastEntryTs()); + } } - } else if (!theCtx.isInitialDataSent()) { - EntityDataUpdate update = new EntityDataUpdate(theCtx.getCmdId(), theCtx.getData(), null, theCtx.getMaxEntitiesPerDataSubscription()); - theCtx.sendWsMsg(update); - theCtx.setInitialDataSent(true); } - } catch (RuntimeException e) { - handleWsCmdRuntimeException(theCtx.getSessionId(), e, cmd); + // Populate with empty values if no data found. + keys.forEach(key -> { + entityData.getAggLatest().putIfAbsent(key.getId(), new ComparisonTsValue(TsValue.EMPTY, TsValue.EMPTY)); + }); + } catch (InterruptedException | ExecutionException e) { + log.warn("[{}][{}][{}] Failed to fetch historical data", ctx.getSessionId(), ctx.getCmdId(), entityData.getEntityId(), e); + ctx.sendWsMsg(new EntityDataUpdate(ctx.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR.getCode(), "Failed to fetch historical data!")); } + }); + ctx.getWsLock().lock(); + try { + EntityDataUpdate update; + if (!ctx.isInitialDataSent()) { + update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null, ctx.getMaxEntitiesPerDataSubscription()); + ctx.setInitialDataSent(true); + } else { + update = new EntityDataUpdate(ctx.getCmdId(), null, entityDataList, ctx.getMaxEntitiesPerDataSubscription()); + } + if (subscribe) { + ctx.createTimeSeriesSubscriptions(lastTsEntityMap, startTs, endTs, true); + } + ctx.sendWsMsg(update); + entityDataList.forEach(ed -> ed.getTimeseries().clear()); + } finally { + ctx.getWsLock().unlock(); } - - @Override - public void onFailure(Throwable t) { - log.warn("[{}][{}] Failed to process command", session.getSessionId(), cmd.getCmdId()); - } + return ctx; }, wsCallBackExecutor); } @@ -416,36 +523,58 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc } private ListenableFuture handleGetTsCmd(TbEntityDataSubCtx ctx, GetTsCmd cmd, boolean subscribe) { + Map queriesKeys = new ConcurrentHashMap<>(); + List keys = cmd.getKeys(); List finalTsKvQueryList; - List tsKvQueryList = cmd.getKeys().stream().map(key -> new BaseReadTsKvQuery( - key, cmd.getStartTs(), cmd.getEndTs(), cmd.getInterval(), getLimit(cmd.getLimit()), cmd.getAgg() - )).collect(Collectors.toList()); + List tsKvQueryList = keys.stream().map(key -> { + var query = new BaseReadTsKvQuery( + key, cmd.getStartTs(), cmd.getEndTs(), cmd.getInterval(), getLimit(cmd.getLimit()), cmd.getAgg() + ); + queriesKeys.put(query.getId(), query.getKey()); + return query; + }).collect(Collectors.toList()); if (cmd.isFetchLatestPreviousPoint()) { finalTsKvQueryList = new ArrayList<>(tsKvQueryList); - finalTsKvQueryList.addAll(cmd.getKeys().stream().map(key -> new BaseReadTsKvQuery( - key, cmd.getStartTs() - TimeUnit.DAYS.toMillis(365), cmd.getStartTs(), cmd.getInterval(), 1, cmd.getAgg() - )).collect(Collectors.toList())); + finalTsKvQueryList.addAll(keys.stream().map(key -> { + var query = new BaseReadTsKvQuery( + key, cmd.getStartTs() - TimeUnit.DAYS.toMillis(365), cmd.getStartTs(), cmd.getInterval(), 1, cmd.getAgg()); + queriesKeys.put(query.getId(), query.getKey()); + return query; + } + ).collect(Collectors.toList())); } else { finalTsKvQueryList = tsKvQueryList; } - Map>> fetchResultMap = new HashMap<>(); - ctx.getData().getData().forEach(entityData -> fetchResultMap.put(entityData, - tsService.findAll(ctx.getTenantId(), entityData.getEntityId(), finalTsKvQueryList))); + Map>> fetchResultMap = new HashMap<>(); + List entityDataList = ctx.getData().getData(); + entityDataList.forEach(entityData -> fetchResultMap.put(entityData, + tsService.findAllByQueries(ctx.getTenantId(), entityData.getEntityId(), finalTsKvQueryList))); return Futures.transform(Futures.allAsList(fetchResultMap.values()), f -> { + // Map that holds last ts for each key for each entity. + Map> lastTsEntityMap = new HashMap<>(); fetchResultMap.forEach((entityData, future) -> { - Map> keyData = new LinkedHashMap<>(); - cmd.getKeys().forEach(key -> keyData.put(key, new ArrayList<>())); try { - List entityTsData = future.get(); - if (entityTsData != null) { - entityTsData.forEach(entry -> keyData.get(entry.getKey()).add(new TsValue(entry.getTs(), entry.getValueAsString()))); + Map lastTsMap = new HashMap<>(); + lastTsEntityMap.put(entityData, lastTsMap); + + List queryResults = future.get(); + if (queryResults != null) { + for (ReadTsKvQueryResult queryResult : queryResults) { + String queryKey = queriesKeys.get(queryResult.getQueryId()); + entityData.getTimeseries().put(queryKey, queryResult.toTsValues()); + lastTsMap.put(queryKey, queryResult.getLastEntryTs()); + } } - keyData.forEach((k, v) -> entityData.getTimeseries().put(k, v.toArray(new TsValue[v.size()]))); + // Populate with empty values if no data found. + keys.forEach(key -> { + if (!entityData.getTimeseries().containsKey(key)) { + entityData.getTimeseries().put(key, new TsValue[0]); + } + }); + if (cmd.isFetchLatestPreviousPoint()) { - entityData.getTimeseries().values().forEach(dataArray -> { - Arrays.sort(dataArray, (o1, o2) -> Long.compare(o2.getTs(), o1.getTs())); - }); + entityData.getTimeseries().values().forEach(dataArray -> Arrays.sort(dataArray, (o1, o2) -> Long.compare(o2.getTs(), o1.getTs()))); } } catch (InterruptedException | ExecutionException e) { log.warn("[{}][{}][{}] Failed to fetch historical data", ctx.getSessionId(), ctx.getCmdId(), entityData.getEntityId(), e); @@ -459,13 +588,13 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null, ctx.getMaxEntitiesPerDataSubscription()); ctx.setInitialDataSent(true); } else { - update = new EntityDataUpdate(ctx.getCmdId(), null, ctx.getData().getData(), ctx.getMaxEntitiesPerDataSubscription()); + update = new EntityDataUpdate(ctx.getCmdId(), null, entityDataList, ctx.getMaxEntitiesPerDataSubscription()); } if (subscribe) { - ctx.createTimeseriesSubscriptions(keys.stream().map(key -> new EntityKey(EntityKeyType.TIME_SERIES, key)).collect(Collectors.toList()), cmd.getStartTs(), cmd.getEndTs()); + ctx.createTimeSeriesSubscriptions(lastTsEntityMap, cmd.getStartTs(), cmd.getEndTs()); } ctx.sendWsMsg(update); - ctx.getData().getData().forEach(ed -> ed.getTimeseries().clear()); + entityDataList.forEach(ed -> ed.getTimeseries().clear()); } finally { ctx.getWsLock().unlock(); } @@ -533,11 +662,7 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc ctx.getWsLock().lock(); try { ctx.createLatestValuesSubscriptions(latestCmd.getKeys()); - if (!ctx.isInitialDataSent()) { - EntityDataUpdate update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null, ctx.getMaxEntitiesPerDataSubscription()); - ctx.sendWsMsg(update); - ctx.setInitialDataSent(true); - } + checkAndSendInitialData(ctx); } finally { ctx.getWsLock().unlock(); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/ReadTsKvQueryInfo.java b/application/src/main/java/org/thingsboard/server/service/subscription/ReadTsKvQueryInfo.java new file mode 100644 index 0000000000..28ca2f6729 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/ReadTsKvQueryInfo.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.Data; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.service.telemetry.cmd.v2.AggKey; + +@Data +public class ReadTsKvQueryInfo { + + private final AggKey key; + private final ReadTsKvQuery query; + private final boolean previous; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAbstractDataSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAbstractDataSubCtx.java index a8283a1e4d..26e97b6f2c 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAbstractDataSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAbstractDataSubCtx.java @@ -86,7 +86,7 @@ public abstract class TbAbstractDataSubCtx newDataMap = newData.getData().stream().collect(Collectors.toMap(EntityData::getEntityId, Function.identity(), (a,b)-> a)); + Map newDataMap = newData.getData().stream().collect(Collectors.toMap(EntityData::getEntityId, Function.identity(), (a, b) -> a)); if (oldDataMap.size() == newDataMap.size() && oldDataMap.keySet().equals(newDataMap.keySet())) { log.trace("[{}][{}] No updates to entity data found", sessionRef.getSessionId(), cmdId); } else { @@ -122,8 +122,17 @@ public abstract class TbAbstractDataSubCtx keys, long startTs, long endTs) { - createSubscriptions(keys, false, startTs, endTs); + public void createTimeSeriesSubscriptions(Map> entityKeyStates, long startTs, long endTs) { + createTimeSeriesSubscriptions(entityKeyStates, startTs, endTs, false); + } + + public void createTimeSeriesSubscriptions(Map> entityKeyStates, long startTs, long endTs, boolean resultToLatestValues) { + entityKeyStates.forEach((entityData, keyStates) -> { + int subIdx = sessionRef.getSessionSubIdSeq().incrementAndGet(); + subToEntityIdMap.put(subIdx, entityData.getEntityId()); + localSubscriptionService.addSubscription( + createTsSub(entityData, subIdx, false, startTs, endTs, keyStates, resultToLatestValues)); + }); } private void createSubscriptions(List keys, boolean latestValues, long startTs, long endTs) { @@ -191,6 +200,14 @@ public abstract class TbAbstractDataSubCtx keyStates) { + return createTsSub(entityData, subIdx, latestValues, startTs, endTs, keyStates, latestValues); + } + + private TbTimeseriesSubscription createTsSub(EntityData entityData, int subIdx, boolean latestValues, long startTs, long endTs, Map keyStates, boolean resultToLatestValues) { log.trace("[{}][{}][{}] Creating time-series subscription for [{}] with keys: {}", serviceId, cmdId, subIdx, entityData.getEntityId(), keyStates); return TbTimeseriesSubscription.builder() .serviceId(serviceId) @@ -198,7 +215,7 @@ public abstract class TbAbstractDataSubCtx sendWsMsg(sessionId, subscriptionUpdate, EntityKeyType.TIME_SERIES, latestValues)) + .updateConsumer((sessionId, subscriptionUpdate) -> sendWsMsg(sessionId, subscriptionUpdate, EntityKeyType.TIME_SERIES, resultToLatestValues)) .allKeys(false) .keyStates(keyStates) .latestValues(latestValues) diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index 888e2c2742..d9beb1ed33 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -716,7 +716,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi "Cmd id is negative value!"); sendWsMsg(sessionRef, update); return false; - } else if (cmd.getQuery() == null && cmd.getLatestCmd() == null && cmd.getHistoryCmd() == null && cmd.getTsCmd() == null) { + } else if (cmd.getQuery() == null && !cmd.hasAnyCmd()) { TelemetrySubscriptionUpdate update = new TelemetrySubscriptionUpdate(cmd.getCmdId(), SubscriptionErrorCode.BAD_REQUEST, "Query is empty!"); sendWsMsg(sessionRef, update); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/AggHistoryCmd.java b/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/AggHistoryCmd.java new file mode 100644 index 0000000000..423c55bd0d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/AggHistoryCmd.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.telemetry.cmd.v2; + +import lombok.Data; + +import java.util.List; + +@Data +public class AggHistoryCmd { + + private List keys; + private long startTs; + private long endTs; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/AggKey.java b/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/AggKey.java new file mode 100644 index 0000000000..d567149d01 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/AggKey.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.telemetry.cmd.v2; + +import lombok.Data; +import org.thingsboard.server.common.data.kv.Aggregation; + +@Data +public class AggKey { + + private int id; + private String key; + private Aggregation agg; + + private Long previousStartTs; + private Long previousEndTs; + private Boolean previousValueOnly; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/AggTimeSeriesCmd.java b/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/AggTimeSeriesCmd.java new file mode 100644 index 0000000000..1f4d88d2c3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/AggTimeSeriesCmd.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.telemetry.cmd.v2; + +import lombok.Data; + +import java.util.List; + +@Data +public class AggTimeSeriesCmd { + + private List keys; + private long startTs; + private long timeWindow; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/EntityDataCmd.java b/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/EntityDataCmd.java index 04335ed9a5..8a187afa7b 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/EntityDataCmd.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/EntityDataCmd.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.telemetry.cmd.v2; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Getter; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -30,17 +31,35 @@ public class EntityDataCmd extends DataCmd { private final LatestValueCmd latestCmd; @Getter private final TimeSeriesCmd tsCmd; + @Getter + private final AggHistoryCmd aggHistoryCmd; + @Getter + private final AggTimeSeriesCmd aggTsCmd; + + public EntityDataCmd(int cmdId, EntityDataQuery query, EntityHistoryCmd historyCmd, LatestValueCmd latestCmd, TimeSeriesCmd tsCmd) { + this(cmdId, query, historyCmd, latestCmd, tsCmd, null, null); + } @JsonCreator public EntityDataCmd(@JsonProperty("cmdId") int cmdId, @JsonProperty("query") EntityDataQuery query, @JsonProperty("historyCmd") EntityHistoryCmd historyCmd, @JsonProperty("latestCmd") LatestValueCmd latestCmd, - @JsonProperty("tsCmd") TimeSeriesCmd tsCmd) { + @JsonProperty("tsCmd") TimeSeriesCmd tsCmd, + @JsonProperty("aggHistoryCmd") AggHistoryCmd aggHistoryCmd, + @JsonProperty("aggTsCmd") AggTimeSeriesCmd aggTsCmd) { super(cmdId); this.query = query; this.historyCmd = historyCmd; this.latestCmd = latestCmd; this.tsCmd = tsCmd; + this.aggHistoryCmd = aggHistoryCmd; + this.aggTsCmd = aggTsCmd; } + + @JsonIgnore + public boolean hasAnyCmd() { + return historyCmd != null || latestCmd != null || tsCmd != null || aggHistoryCmd != null || aggTsCmd != null; + } + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java index b8688b1524..cf17eec88d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -32,6 +33,8 @@ import java.util.List; */ public interface TimeseriesService { + ListenableFuture> findAllByQueries(TenantId tenantId, EntityId entityId, List queries); + ListenableFuture> findAll(TenantId tenantId, EntityId entityId, List queries); ListenableFuture> findLatest(TenantId tenantId, EntityId entityId, Collection keys); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/AggTsKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/AggTsKvEntry.java new file mode 100644 index 0000000000..85330d3577 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/AggTsKvEntry.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.kv; + +import lombok.ToString; +import org.thingsboard.server.common.data.query.TsValue; + +@ToString +public class AggTsKvEntry extends BasicTsKvEntry { + + private static final long serialVersionUID = -1933884317450255935L; + + private final long count; + + public AggTsKvEntry(long ts, KvEntry kv, long count) { + super(ts, kv); + this.count = count; + } + + @Override + public TsValue toTsValue() { + return new TsValue(ts, getValueAsString(), count); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java index fc111da8e2..29b98d5e83 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java @@ -31,8 +31,7 @@ public class BaseReadTsKvQuery extends BaseTsKvQuery implements ReadTsKvQuery { this(key, startTs, endTs, interval, limit, aggregation, "DESC"); } - public BaseReadTsKvQuery(String key, long startTs, long endTs, long interval, int limit, Aggregation aggregation, - String order) { + public BaseReadTsKvQuery(String key, long startTs, long endTs, long interval, int limit, Aggregation aggregation, String order) { super(key, startTs, endTs); this.interval = interval; this.limit = limit; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseTsKvQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseTsKvQuery.java index 4102b30a6f..245af53d55 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseTsKvQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseTsKvQuery.java @@ -20,11 +20,16 @@ import lombok.Data; @Data public class BaseTsKvQuery implements TsKvQuery { + private static final ThreadLocal idSeq = ThreadLocal.withInitial(() -> 0); + + private final int id; private final String key; private final long startTs; private final long endTs; public BaseTsKvQuery(String key, long startTs, long endTs) { + this.id = idSeq.get(); + idSeq.set(id + 1); this.key = key; this.startTs = startTs; this.endTs = endTs; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java index 6fc32e627b..e5a7ac65e0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java @@ -20,7 +20,7 @@ import java.util.Optional; public class BasicTsKvEntry implements TsKvEntry { private static final int MAX_CHARS_PER_DATA_POINT = 512; - private final long ts; + protected final long ts; private final KvEntry kv; public BasicTsKvEntry(long ts, KvEntry kv) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/ReadTsKvQueryResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/ReadTsKvQueryResult.java new file mode 100644 index 0000000000..9c7a2f1754 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/ReadTsKvQueryResult.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.kv; + +import lombok.Data; +import org.thingsboard.server.common.data.query.TsValue; + +import java.util.ArrayList; +import java.util.List; + +@Data +public class ReadTsKvQueryResult { + + private final int queryId; + // Holds the data list; + private final List data; + // Holds the max ts of the records that match aggregation intervals (not the ts of the aggregation window, but the ts of the last record among all the intervals) + private final long lastEntryTs; + + public TsValue[] toTsValues() { + if (data != null && !data.isEmpty()) { + List queryValues = new ArrayList<>(); + for (TsKvEntry v : data) { + queryValues.add(v.toTsValue()); // TODO: add count here. + } + return queryValues.toArray(new TsValue[queryValues.size()]); + } else { + return new TsValue[0]; + } + } + + public TsValue toTsValue() { + if (data == null || data.isEmpty()) { + return TsValue.EMPTY; + } + if (data.size() > 1) { + throw new RuntimeException("Query Result has multiple data points!"); + } + return data.get(0).toTsValue(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java index 20d25d5399..06ac0d84a3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java @@ -16,10 +16,11 @@ package org.thingsboard.server.common.data.kv; import com.fasterxml.jackson.annotation.JsonIgnore; +import org.thingsboard.server.common.data.query.TsValue; /** * Represents time series KV data entry - * + * * @author ashvayka * */ @@ -30,4 +31,9 @@ public interface TsKvEntry extends KvEntry { @JsonIgnore int getDataPoints(); + @JsonIgnore + default TsValue toTsValue() { + return new TsValue(getTs(), getValueAsString()); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntryAggWrapper.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntryAggWrapper.java new file mode 100644 index 0000000000..2825008485 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntryAggWrapper.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.kv; + +import lombok.Data; + +@Data +public class TsKvEntryAggWrapper { + + private final TsKvEntry entry; + private final long lastEntryTs; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvQuery.java index cf01c322a5..78bf01c0e9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvQuery.java @@ -17,6 +17,8 @@ package org.thingsboard.server.common.data.kv; public interface TsKvQuery { + int getId(); + String getKey(); long getStartTs(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/ComparisonTsValue.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/ComparisonTsValue.java new file mode 100644 index 0000000000..301fbdb0ee --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/ComparisonTsValue.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.query; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class ComparisonTsValue { + + private TsValue current; + private TsValue previous; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java index c0cc778280..2b24866ac4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java @@ -16,15 +16,21 @@ package org.thingsboard.server.common.data.query; import lombok.Data; +import lombok.RequiredArgsConstructor; import org.thingsboard.server.common.data.id.EntityId; import java.util.Map; @Data +@RequiredArgsConstructor public class EntityData { private final EntityId entityId; private final Map> latest; private final Map timeseries; + private final Map aggLatest; + public EntityData(EntityId entityId, Map> latest, Map timeseries) { + this(entityId, latest, timeseries, null); + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityKey.java index 5b4b5cd257..6aacc90060 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityKey.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityKey.java @@ -23,6 +23,8 @@ import java.io.Serializable; @ApiModel @Data public class EntityKey implements Serializable { + private static final long serialVersionUID = -6421575477523085543L; + private final EntityKeyType type; private final String key; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/TsValue.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/TsValue.java index a086f1c21f..9dbfdfbb28 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/TsValue.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/TsValue.java @@ -15,11 +15,23 @@ */ package org.thingsboard.server.common.data.query; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.Data; +import lombok.RequiredArgsConstructor; @Data +@RequiredArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) public class TsValue { + public static final TsValue EMPTY = new TsValue(0, ""); + private final long ts; private final String value; + private final Long count; + + public TsValue(long ts, String value) { + this(ts, value, null); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 171417402d..e27bf174a9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -631,7 +631,7 @@ public class ModelConstants { protected static final String[] NONE_AGGREGATION_COLUMNS = new String[]{LONG_VALUE_COLUMN, DOUBLE_VALUE_COLUMN, BOOLEAN_VALUE_COLUMN, STRING_VALUE_COLUMN, JSON_VALUE_COLUMN, KEY_COLUMN, TS_COLUMN}; - protected static final String[] COUNT_AGGREGATION_COLUMNS = new String[]{count(LONG_VALUE_COLUMN), count(DOUBLE_VALUE_COLUMN), count(BOOLEAN_VALUE_COLUMN), count(STRING_VALUE_COLUMN), count(JSON_VALUE_COLUMN)}; + protected static final String[] COUNT_AGGREGATION_COLUMNS = new String[]{count(LONG_VALUE_COLUMN), count(DOUBLE_VALUE_COLUMN), count(BOOLEAN_VALUE_COLUMN), count(STRING_VALUE_COLUMN), count(JSON_VALUE_COLUMN), max(TS_COLUMN)}; protected static final String[] MIN_AGGREGATION_COLUMNS = ArrayUtils.addAll(COUNT_AGGREGATION_COLUMNS, new String[]{min(LONG_VALUE_COLUMN), min(DOUBLE_VALUE_COLUMN), min(BOOLEAN_VALUE_COLUMN), min(STRING_VALUE_COLUMN), min(JSON_VALUE_COLUMN)}); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractTsKvEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractTsKvEntity.java index 1824c96247..5a5985d125 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractTsKvEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractTsKvEntity.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.model.sql; import lombok.Data; +import org.thingsboard.server.common.data.kv.AggTsKvEntry; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; @@ -80,6 +81,18 @@ public abstract class AbstractTsKvEntity implements ToData { @Transient protected String strKey; + @Transient + protected Long aggValuesLastTs; + @Transient + protected Long aggValuesCount; + + public AbstractTsKvEntity() { + } + + public AbstractTsKvEntity(Long aggValuesLastTs) { + this.aggValuesLastTs = aggValuesLastTs; + } + public abstract boolean isNotEmpty(); protected static boolean isAllNull(Object... args) { @@ -105,7 +118,12 @@ public abstract class AbstractTsKvEntity implements ToData { } else if (jsonValue != null) { kvEntry = new JsonDataEntry(strKey, jsonValue); } - return new BasicTsKvEntry(ts, kvEntry); + + if (aggValuesCount == null) { + return new BasicTsKvEntry(ts, kvEntry); + } else { + return new AggTsKvEntry(ts, kvEntry, aggValuesCount); + } } } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/ts/TimescaleTsKvEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/ts/TimescaleTsKvEntity.java index 88c1693e10..ba5ad92f4f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/ts/TimescaleTsKvEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/ts/TimescaleTsKvEntity.java @@ -62,6 +62,7 @@ import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.F @ColumnResult(name = "doubleCountValue", type = Long.class), @ColumnResult(name = "strValue", type = String.class), @ColumnResult(name = "aggType", type = String.class), + @ColumnResult(name = "maxAggTs", type = Long.class), } ), }), @@ -78,6 +79,7 @@ import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.F @ColumnResult(name = "longValueCount", type = Long.class), @ColumnResult(name = "doubleValueCount", type = Long.class), @ColumnResult(name = "jsonValueCount", type = Long.class), + @ColumnResult(name = "maxAggTs", type = Long.class), } ) }), @@ -114,7 +116,8 @@ public final class TimescaleTsKvEntity extends AbstractTsKvEntity { public TimescaleTsKvEntity() { } - public TimescaleTsKvEntity(Long tsBucket, Long interval, Long longValue, Double doubleValue, Long longCountValue, Long doubleCountValue, String strValue, String aggType) { + public TimescaleTsKvEntity(Long tsBucket, Long interval, Long longValue, Double doubleValue, Long longCountValue, Long doubleCountValue, String strValue, String aggType, Long aggValuesLastTs) { + super(aggValuesLastTs); if (!StringUtils.isEmpty(strValue)) { this.strValue = strValue; } @@ -135,6 +138,7 @@ public final class TimescaleTsKvEntity extends AbstractTsKvEntity { } else { this.doubleValue = 0.0; } + this.aggValuesCount = totalCount; break; case SUM: if (doubleCountValue > 0) { @@ -157,7 +161,8 @@ public final class TimescaleTsKvEntity extends AbstractTsKvEntity { } } - public TimescaleTsKvEntity(Long tsBucket, Long interval, Long booleanValueCount, Long strValueCount, Long longValueCount, Long doubleValueCount, Long jsonValueCount) { + public TimescaleTsKvEntity(Long tsBucket, Long interval, Long booleanValueCount, Long strValueCount, Long longValueCount, Long doubleValueCount, Long jsonValueCount, Long aggValuesLastTs) { + super(aggValuesLastTs); if (!isAllNull(tsBucket, interval, booleanValueCount, strValueCount, longValueCount, doubleValueCount, jsonValueCount)) { this.ts = tsBucket + interval / 2; if (booleanValueCount != 0) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvEntity.java index 1c0277f8b6..15fd30c742 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvEntity.java @@ -16,12 +16,15 @@ package org.thingsboard.server.dao.model.sqlts.ts; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; import javax.persistence.Entity; import javax.persistence.IdClass; import javax.persistence.Table; +import javax.persistence.Transient; +@EqualsAndHashCode(callSuper = true) @Data @Entity @Table(name = "ts_kv") @@ -31,11 +34,13 @@ public final class TsKvEntity extends AbstractTsKvEntity { public TsKvEntity() { } - public TsKvEntity(String strValue) { + public TsKvEntity(String strValue, Long aggValuesLastTs) { + super(aggValuesLastTs); this.strValue = strValue; } - public TsKvEntity(Long longValue, Double doubleValue, Long longCountValue, Long doubleCountValue, String aggType) { + public TsKvEntity(Long longValue, Double doubleValue, Long longCountValue, Long doubleCountValue, String aggType, Long aggValuesLastTs) { + super(aggValuesLastTs); if (!isAllNull(longValue, doubleValue, longCountValue, doubleCountValue)) { switch (aggType) { case AVG: @@ -52,6 +57,7 @@ public final class TsKvEntity extends AbstractTsKvEntity { } else { this.doubleValue = 0.0; } + this.aggValuesCount = totalCount; break; case SUM: if (doubleCountValue > 0) { @@ -74,7 +80,8 @@ public final class TsKvEntity extends AbstractTsKvEntity { } } - public TsKvEntity(Long booleanValueCount, Long strValueCount, Long longValueCount, Long doubleValueCount, Long jsonValueCount) { + public TsKvEntity(Long booleanValueCount, Long strValueCount, Long longValueCount, Long doubleValueCount, Long jsonValueCount, Long aggValuesLastTs) { + super(aggValuesLastTs); if (!isAllNull(booleanValueCount, strValueCount, longValueCount, doubleValueCount)) { if (booleanValueCount != 0) { this.longValue = booleanValueCount; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityDataAdapter.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityDataAdapter.java index c30ecdd9a2..e2d691f32b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityDataAdapter.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityDataAdapter.java @@ -54,8 +54,8 @@ public class EntityDataAdapter { EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); Map> latest = new HashMap<>(); - Map timeseries = new HashMap<>(); - EntityData entityData = new EntityData(entityId, latest, timeseries); + //Maybe avoid empty hashmaps? + EntityData entityData = new EntityData(entityId, latest, new HashMap<>(), new HashMap<>()); for (EntityKeyMapping mapping : selectionMapping) { if (!mapping.isIgnore()) { EntityKey entityKey = mapping.getEntityKey(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 69f16d4bbc..22c4661fcd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -17,8 +17,6 @@ package org.thingsboard.server.dao.sqlts; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; @@ -26,9 +24,9 @@ import org.springframework.data.domain.Sort; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; @@ -47,10 +45,9 @@ import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.CompletableFuture; import java.util.function.Function; -import java.util.stream.Collectors; +@SuppressWarnings("UnstableApiUsage") @Slf4j public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSqlTimeseriesDao implements TimeseriesDao { @@ -81,7 +78,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq Comparator.comparing((Function) AbstractTsKvEntity::getEntityId) .thenComparing(AbstractTsKvEntity::getKey) .thenComparing(AbstractTsKvEntity::getTs) - ); + ); } @PreDestroy @@ -114,32 +111,32 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq } @Override - public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { return processFindAllAsync(tenantId, entityId, queries); } @Override - public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + public ListenableFuture findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { if (query.getAggregation() == Aggregation.NONE) { - return findAllAsyncWithLimit(entityId, query); + return Futures.immediateFuture(findAllAsyncWithLimit(entityId, query)); } else { - List>> futures = new ArrayList<>(); - long endPeriod = query.getEndTs(); + List>> futures = new ArrayList<>(); long startPeriod = query.getStartTs(); + long endPeriod = Math.max(query.getStartTs() + 1, query.getEndTs()); long step = query.getInterval(); - while (startPeriod <= endPeriod) { + while (startPeriod < endPeriod) { long startTs = startPeriod; - long endTs = Math.min(startPeriod + step, endPeriod + 1); + long endTs = Math.min(startPeriod + step, endPeriod); long ts = startTs + (endTs - startTs) / 2; - ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation()); + ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation()); futures.add(aggregateTsKvEntry); startPeriod = endTs; } - return getTskvEntriesFuture(Futures.allAsList(futures)); + return getReadTsKvQueryResultFuture(query, Futures.allAsList(futures)); } } - private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { + private ReadTsKvQueryResult findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { Integer keyId = getOrSaveKeyId(query.getKey()); List tsKvEntities = tsKvRepository.findAllWithLimit( entityId.getId(), @@ -147,125 +144,52 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq query.getStartTs(), query.getEndTs(), PageRequest.of(0, query.getLimit(), - Sort.by(new Sort.Order(Sort.Direction.fromString(query.getOrder()), "ts").nullsNative()))); + Sort.by(new Sort.Order(Sort.Direction.fromString(query.getOrder()), "ts").nullsNative()))); tsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(query.getKey())); - return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); + List tsKvEntries = DaoUtil.convertDataList(tsKvEntities); + long lastTs = tsKvEntries.stream().map(TsKvEntry::getTs).max(Long::compare).orElse(query.getStartTs()); + return new ReadTsKvQueryResult(query.getId(), tsKvEntries, lastTs); } - ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { - List> entitiesFutures = new ArrayList<>(); - switchAggregation(entityId, key, startTs, endTs, aggregation, entitiesFutures); - return Futures.transform(setFutures(entitiesFutures), entity -> { + ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { + return service.submit(() -> { + TsKvEntity entity = switchAggregation(entityId, key, startTs, endTs, aggregation); if (entity != null && entity.isNotEmpty()) { entity.setEntityId(entityId.getId()); entity.setStrKey(key); entity.setTs(ts); - return Optional.of(DaoUtil.getData(entity)); + return Optional.of(entity); } else { return Optional.empty(); } - }, MoreExecutors.directExecutor()); + }); } - protected void switchAggregation(EntityId entityId, String key, long startTs, long endTs, Aggregation aggregation, List> entitiesFutures) { + protected TsKvEntity switchAggregation(EntityId entityId, String key, long startTs, long endTs, Aggregation aggregation) { + var keyId = getOrSaveKeyId(key); switch (aggregation) { case AVG: - findAvg(entityId, key, startTs, endTs, entitiesFutures); - break; + return tsKvRepository.findAvg(entityId.getId(), keyId, startTs, endTs); case MAX: - findMax(entityId, key, startTs, endTs, entitiesFutures); - break; + var max = tsKvRepository.findNumericMax(entityId.getId(), keyId, startTs, endTs); + if (max.isNotEmpty()) { + return max; + } else { + return tsKvRepository.findStringMax(entityId.getId(), keyId, startTs, endTs); + } case MIN: - findMin(entityId, key, startTs, endTs, entitiesFutures); - break; + var min = tsKvRepository.findNumericMin(entityId.getId(), keyId, startTs, endTs); + if (min.isNotEmpty()) { + return min; + } else { + return tsKvRepository.findStringMin(entityId.getId(), keyId, startTs, endTs); + } case SUM: - findSum(entityId, key, startTs, endTs, entitiesFutures); - break; + return tsKvRepository.findSum(entityId.getId(), keyId, startTs, endTs); case COUNT: - findCount(entityId, key, startTs, endTs, entitiesFutures); - break; + return tsKvRepository.findCount(entityId.getId(), keyId, startTs, endTs); default: throw new IllegalArgumentException("Not supported aggregation type: " + aggregation); } } - - protected void findCount(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures) { - Integer keyId = getOrSaveKeyId(key); - entitiesFutures.add(tsKvRepository.findCount( - entityId.getId(), - keyId, - startTs, - endTs)); - } - - protected void findSum(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures) { - Integer keyId = getOrSaveKeyId(key); - entitiesFutures.add(tsKvRepository.findSum( - entityId.getId(), - keyId, - startTs, - endTs)); - } - - protected void findMin(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures) { - Integer keyId = getOrSaveKeyId(key); - entitiesFutures.add(tsKvRepository.findStringMin( - entityId.getId(), - keyId, - startTs, - endTs)); - entitiesFutures.add(tsKvRepository.findNumericMin( - entityId.getId(), - keyId, - startTs, - endTs)); - } - - protected void findMax(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures) { - Integer keyId = getOrSaveKeyId(key); - entitiesFutures.add(tsKvRepository.findStringMax( - entityId.getId(), - keyId, - startTs, - endTs)); - entitiesFutures.add(tsKvRepository.findNumericMax( - entityId.getId(), - keyId, - startTs, - endTs)); - } - - protected void findAvg(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures) { - Integer keyId = getOrSaveKeyId(key); - entitiesFutures.add(tsKvRepository.findAvg( - entityId.getId(), - keyId, - startTs, - endTs)); - } - - protected SettableFuture setFutures(List> entitiesFutures) { - SettableFuture listenableFuture = SettableFuture.create(); - CompletableFuture> entities = - CompletableFuture.allOf(entitiesFutures.toArray(new CompletableFuture[entitiesFutures.size()])) - .thenApply(v -> entitiesFutures.stream() - .map(CompletableFuture::join) - .collect(Collectors.toList())); - - entities.whenComplete((tsKvEntities, throwable) -> { - if (throwable != null) { - listenableFuture.setException(throwable); - } else { - TsKvEntity result = null; - for (TsKvEntity entity : tsKvEntities) { - if (entity.isNotEmpty()) { - result = entity; - break; - } - } - listenableFuture.set(result); - } - }); - return listenableFuture; - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index aa0d9b1c47..3de21e41bf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -24,6 +24,7 @@ import org.springframework.beans.factory.annotation.Value; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; @@ -38,6 +39,7 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +@SuppressWarnings("UnstableApiUsage") @Slf4j public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseriesDao implements AggregationTimeseriesDao { @@ -86,22 +88,19 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries } } - protected ListenableFuture> processFindAllAsync(TenantId tenantId, EntityId entityId, List queries) { - List>> futures = queries + protected ListenableFuture> processFindAllAsync(TenantId tenantId, EntityId entityId, List queries) { + List> futures = queries .stream() .map(query -> findAllAsync(tenantId, entityId, query)) .collect(Collectors.toList()); return Futures.transform(Futures.allAsList(futures), new Function<>() { @Nullable @Override - public List apply(@Nullable List> results) { + public List apply(@Nullable List results) { if (results == null || results.isEmpty()) { return null; } - return results.stream() - .filter(Objects::nonNull) - .flatMap(List::stream) - .collect(Collectors.toList()); + return results.stream().filter(Objects::nonNull).collect(Collectors.toList()); } }, service); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java index 31270bacc7..994555aaa5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java @@ -19,11 +19,12 @@ import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; public interface AggregationTimeseriesDao { - ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query); + ListenableFuture findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query); } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java index 82e54168bc..1946198505 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java @@ -22,14 +22,20 @@ import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DataIntegrityViolationException; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; +import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; import javax.annotation.Nullable; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -80,18 +86,20 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni return keyId; } - protected ListenableFuture> getTskvEntriesFuture(ListenableFuture>> future) { - return Futures.transform(future, new Function>, List>() { + protected ListenableFuture getReadTsKvQueryResultFuture(ReadTsKvQuery query, ListenableFuture>> future) { + return Futures.transform(future, new Function<>() { @Nullable @Override - public List apply(@Nullable List> results) { + public ReadTsKvQueryResult apply(@Nullable List> results) { if (results == null || results.isEmpty()) { return null; } - return results.stream() - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toList()); + List data = results.stream().filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList()); + var lastTs = data.stream().map(AbstractTsKvEntity::getAggValuesLastTs).filter(Objects::nonNull).max(Long::compare); + if (lastTs.isEmpty()) { + lastTs = data.stream().map(AbstractTsKvEntity::getTs).filter(Objects::nonNull).max(Long::compare); + } + return new ReadTsKvQueryResult(query.getId(), DaoUtil.convertDataList(data), lastTs.orElse(query.getStartTs())); } }, service); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index db30adead2..50975f21de 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; @@ -190,7 +191,8 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme long endTs = query.getStartTs() - 1; ReadTsKvQuery findNewLatestQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, endTs - startTs, 1, Aggregation.NONE, DESC_ORDER); - return aggregationTimeseriesDao.findAllAsync(tenantId, entityId, findNewLatestQuery); + return Futures.transform(aggregationTimeseriesDao.findAllAsync(tenantId, entityId, findNewLatestQuery), + ReadTsKvQueryResult::getData, MoreExecutors.directExecutor()); } protected ListenableFuture getFindLatestFuture(EntityId entityId, String key) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java index 9cf796b0aa..d37234dfde 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java @@ -36,54 +36,79 @@ public class AggregationRepository { public static final String FIND_SUM = "findSum"; public static final String FIND_COUNT = "findCount"; - public static final String FROM_WHERE_CLAUSE = "FROM ts_kv tskv WHERE tskv.entity_id = cast(:entityId AS uuid) AND tskv.key= cast(:entityKey AS int) AND tskv.ts > :startTs AND tskv.ts <= :endTs GROUP BY tskv.entity_id, tskv.key, tsBucket ORDER BY tskv.entity_id, tskv.key, tsBucket"; - - public static final String FIND_AVG_QUERY = "SELECT time_bucket(:timeBucket, tskv.ts, :startTs) AS tsBucket, :timeBucket AS interval, SUM(COALESCE(tskv.long_v, 0)) AS longValue, SUM(COALESCE(tskv.dbl_v, 0.0)) AS doubleValue, SUM(CASE WHEN tskv.long_v IS NULL THEN 0 ELSE 1 END) AS longCountValue, SUM(CASE WHEN tskv.dbl_v IS NULL THEN 0 ELSE 1 END) AS doubleCountValue, null AS strValue, 'AVG' AS aggType "; - - public static final String FIND_MAX_QUERY = "SELECT time_bucket(:timeBucket, tskv.ts, :startTs) AS tsBucket, :timeBucket AS interval, MAX(COALESCE(tskv.long_v, -9223372036854775807)) AS longValue, MAX(COALESCE(tskv.dbl_v, -1.79769E+308)) as doubleValue, SUM(CASE WHEN tskv.long_v IS NULL THEN 0 ELSE 1 END) AS longCountValue, SUM(CASE WHEN tskv.dbl_v IS NULL THEN 0 ELSE 1 END) AS doubleCountValue, MAX(tskv.str_v) AS strValue, 'MAX' AS aggType "; - - public static final String FIND_MIN_QUERY = "SELECT time_bucket(:timeBucket, tskv.ts, :startTs) AS tsBucket, :timeBucket AS interval, MIN(COALESCE(tskv.long_v, 9223372036854775807)) AS longValue, MIN(COALESCE(tskv.dbl_v, 1.79769E+308)) as doubleValue, SUM(CASE WHEN tskv.long_v IS NULL THEN 0 ELSE 1 END) AS longCountValue, SUM(CASE WHEN tskv.dbl_v IS NULL THEN 0 ELSE 1 END) AS doubleCountValue, MIN(tskv.str_v) AS strValue, 'MIN' AS aggType "; - - public static final String FIND_SUM_QUERY = "SELECT time_bucket(:timeBucket, tskv.ts, :startTs) AS tsBucket, :timeBucket AS interval, SUM(COALESCE(tskv.long_v, 0)) AS longValue, SUM(COALESCE(tskv.dbl_v, 0.0)) AS doubleValue, SUM(CASE WHEN tskv.long_v IS NULL THEN 0 ELSE 1 END) AS longCountValue, SUM(CASE WHEN tskv.dbl_v IS NULL THEN 0 ELSE 1 END) AS doubleCountValue, null AS strValue, null AS jsonValue, 'SUM' AS aggType "; - - public static final String FIND_COUNT_QUERY = "SELECT time_bucket(:timeBucket, tskv.ts, :startTs) AS tsBucket, :timeBucket AS interval, SUM(CASE WHEN tskv.bool_v IS NULL THEN 0 ELSE 1 END) AS booleanValueCount, SUM(CASE WHEN tskv.str_v IS NULL THEN 0 ELSE 1 END) AS strValueCount, SUM(CASE WHEN tskv.long_v IS NULL THEN 0 ELSE 1 END) AS longValueCount, SUM(CASE WHEN tskv.dbl_v IS NULL THEN 0 ELSE 1 END) AS doubleValueCount, SUM(CASE WHEN tskv.json_v IS NULL THEN 0 ELSE 1 END) AS jsonValueCount "; + public static final String FROM_WHERE_CLAUSE = "FROM ts_kv tskv WHERE " + + "tskv.entity_id = cast(:entityId AS uuid) " + + "AND tskv.key= cast(:entityKey AS int) " + + "AND tskv.ts >= :startTs AND tskv.ts < :endTs " + + "GROUP BY tskv.entity_id, tskv.key, tsBucket " + + "ORDER BY tskv.entity_id, tskv.key, tsBucket"; + + public static final String FIND_AVG_QUERY = "SELECT " + + "time_bucket(:timeBucket, tskv.ts, :startTs) AS tsBucket, :timeBucket AS interval, " + + "SUM(COALESCE(tskv.long_v, 0)) AS longValue, " + + "SUM(COALESCE(tskv.dbl_v, 0.0)) AS doubleValue, " + + "SUM(CASE WHEN tskv.long_v IS NULL THEN 0 ELSE 1 END) AS longCountValue, " + + "SUM(CASE WHEN tskv.dbl_v IS NULL THEN 0 ELSE 1 END) AS doubleCountValue, " + + "null AS strValue, 'AVG' AS aggType, MAX(tskv.ts) AS maxAggTs "; + + public static final String FIND_MAX_QUERY = "SELECT " + + "time_bucket(:timeBucket, tskv.ts, :startTs) AS tsBucket, :timeBucket AS interval, " + + "MAX(COALESCE(tskv.long_v, -9223372036854775807)) AS longValue, " + + "MAX(COALESCE(tskv.dbl_v, -1.79769E+308)) as doubleValue, " + + "SUM(CASE WHEN tskv.long_v IS NULL THEN 0 ELSE 1 END) AS longCountValue, " + + "SUM(CASE WHEN tskv.dbl_v IS NULL THEN 0 ELSE 1 END) AS doubleCountValue, " + + "MAX(tskv.str_v) AS strValue, 'MAX' AS aggType, MAX(tskv.ts) AS maxAggTs "; + + public static final String FIND_MIN_QUERY = "SELECT " + + "time_bucket(:timeBucket, tskv.ts, :startTs) AS tsBucket, :timeBucket AS interval, " + + "MIN(COALESCE(tskv.long_v, 9223372036854775807)) AS longValue, " + + "MIN(COALESCE(tskv.dbl_v, 1.79769E+308)) as doubleValue, " + + "SUM(CASE WHEN tskv.long_v IS NULL THEN 0 ELSE 1 END) AS longCountValue, " + + "SUM(CASE WHEN tskv.dbl_v IS NULL THEN 0 ELSE 1 END) AS doubleCountValue, " + + "MIN(tskv.str_v) AS strValue, 'MIN' AS aggType, MAX(tskv.ts) AS maxAggTs "; + + public static final String FIND_SUM_QUERY = "SELECT " + + "time_bucket(:timeBucket, tskv.ts, :startTs) AS tsBucket, :timeBucket AS interval, " + + "SUM(COALESCE(tskv.long_v, 0)) AS longValue, SUM(COALESCE(tskv.dbl_v, 0.0)) AS doubleValue, " + + "SUM(CASE WHEN tskv.long_v IS NULL THEN 0 ELSE 1 END) AS longCountValue, " + + "SUM(CASE WHEN tskv.dbl_v IS NULL THEN 0 ELSE 1 END) AS doubleCountValue, " + + "null AS strValue, null AS jsonValue, 'SUM' AS aggType, MAX(tskv.ts) AS maxAggTs "; + + public static final String FIND_COUNT_QUERY = "SELECT " + + "time_bucket(:timeBucket, tskv.ts, :startTs) AS tsBucket, :timeBucket AS interval, " + + "SUM(CASE WHEN tskv.bool_v IS NULL THEN 0 ELSE 1 END) AS booleanValueCount, " + + "SUM(CASE WHEN tskv.str_v IS NULL THEN 0 ELSE 1 END) AS strValueCount, " + + "SUM(CASE WHEN tskv.long_v IS NULL THEN 0 ELSE 1 END) AS longValueCount, " + + "SUM(CASE WHEN tskv.dbl_v IS NULL THEN 0 ELSE 1 END) AS doubleValueCount, " + + "SUM(CASE WHEN tskv.json_v IS NULL THEN 0 ELSE 1 END) AS jsonValueCount, " + + "MAX(tskv.ts) AS maxAggTs "; @PersistenceContext private EntityManager entityManager; - @Async - public CompletableFuture> findAvg(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { - @SuppressWarnings("unchecked") - List resultList = getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_AVG); - return CompletableFuture.supplyAsync(() -> resultList); + @SuppressWarnings("unchecked") + public List findAvg(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { + return getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_AVG); } - @Async - public CompletableFuture> findMax(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { - @SuppressWarnings("unchecked") - List resultList = getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_MAX); - return CompletableFuture.supplyAsync(() -> resultList); + @SuppressWarnings("unchecked") + public List findMax(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { + return getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_MAX); } - @Async - public CompletableFuture> findMin(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { - @SuppressWarnings("unchecked") - List resultList = getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_MIN); - return CompletableFuture.supplyAsync(() -> resultList); + @SuppressWarnings("unchecked") + public List findMin(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { + return getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_MIN); } - @Async - public CompletableFuture> findSum(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { - @SuppressWarnings("unchecked") - List resultList = getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_SUM); - return CompletableFuture.supplyAsync(() -> resultList); + @SuppressWarnings("unchecked") + public List findSum(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { + return getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_SUM); } - @Async - public CompletableFuture> findCount(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { - @SuppressWarnings("unchecked") - List resultList = getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_COUNT); - return CompletableFuture.supplyAsync(() -> resultList); + @SuppressWarnings("unchecked") + public List findCount(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { + return getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_COUNT); } private List getResultList(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs, String query) { @@ -96,5 +121,4 @@ public class AggregationRepository { .getResultList(); } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 9c03a9e3dd..6e9f9e61b8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -30,11 +30,13 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity; +import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; import org.thingsboard.server.dao.sqlts.AbstractSqlTimeseriesDao; @@ -101,13 +103,13 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements } @Override - public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { return processFindAllAsync(tenantId, entityId, queries); } @Override public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - int dataPointDays = getDataPointDays(tsKvEntry, computeTtl(ttl)); + int dataPointDays = getDataPointDays(tsKvEntry, computeTtl(ttl)); String strKey = tsKvEntry.getKey(); Integer keyId = getOrSaveKeyId(strKey); TimescaleTsKvEntity entity = new TimescaleTsKvEntity(); @@ -148,15 +150,15 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements } @Override - public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + public ListenableFuture findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { if (query.getAggregation() == Aggregation.NONE) { - return findAllAsyncWithLimit(entityId, query); + return Futures.immediateFuture(findAllAsyncWithLimit(entityId, query)); } else { long startTs = query.getStartTs(); - long endTs = query.getEndTs(); + long endTs = Math.max(query.getStartTs() + 1, query.getEndTs()); long timeBucket = query.getInterval(); - ListenableFuture>> future = findAllAndAggregateAsync(entityId, query.getKey(), startTs, endTs, timeBucket, query.getAggregation()); - return getTskvEntriesFuture(future); + List> data = findAllAndAggregateAsync(entityId, query.getKey(), startTs, endTs, timeBucket, query.getAggregation()); + return getReadTsKvQueryResultFuture(query, Futures.immediateFuture(data)); } } @@ -165,7 +167,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements super.cleanup(systemTtl); } - private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { + private ReadTsKvQueryResult findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { String strKey = query.getKey(); Integer keyId = getOrSaveKeyId(strKey); List timescaleTsKvEntities = tsKvRepository.findAllWithLimit( @@ -174,105 +176,60 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements query.getStartTs(), query.getEndTs(), PageRequest.of(0, query.getLimit(), - Sort.by(new Sort.Order(Sort.Direction.fromString(query.getOrder()), "ts").nullsNative())));; + Sort.by(new Sort.Order(Sort.Direction.fromString(query.getOrder()), "ts").nullsNative()))); + ; timescaleTsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(strKey)); - return Futures.immediateFuture(DaoUtil.convertDataList(timescaleTsKvEntities)); + var tsKvEntries = DaoUtil.convertDataList(timescaleTsKvEntities); + long lastTs = tsKvEntries.stream().map(TsKvEntry::getTs).max(Long::compare).orElse(query.getStartTs()); + return new ReadTsKvQueryResult(query.getId(), tsKvEntries, lastTs); } - private ListenableFuture>> findAllAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long timeBucket, Aggregation aggregation) { - CompletableFuture> listCompletableFuture = switchAggregation(key, startTs, endTs, timeBucket, aggregation, entityId.getId()); - SettableFuture> listenableFuture = SettableFuture.create(); - listCompletableFuture.whenComplete((timescaleTsKvEntities, throwable) -> { - if (throwable != null) { - listenableFuture.setException(throwable); - } else { - listenableFuture.set(timescaleTsKvEntities); - } - }); - return Futures.transform(listenableFuture, timescaleTsKvEntities -> { - if (!CollectionUtils.isEmpty(timescaleTsKvEntities)) { - List> result = new ArrayList<>(); - timescaleTsKvEntities.forEach(entity -> { - if (entity != null && entity.isNotEmpty()) { - entity.setEntityId(entityId.getId()); - entity.setStrKey(key); - result.add(Optional.of(DaoUtil.getData(entity))); - } else { - result.add(Optional.empty()); - } - }); - return result; - } else { - return Collections.emptyList(); - } - }, MoreExecutors.directExecutor()); + private List> findAllAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long timeBucket, Aggregation aggregation) { + long interval = endTs - startTs; + long remainingPart = interval % timeBucket; + List timescaleTsKvEntities; + if (remainingPart == 0) { + timescaleTsKvEntities = switchAggregation(key, startTs, endTs, timeBucket, aggregation, entityId.getId()); + } else { + interval = interval - remainingPart; + timescaleTsKvEntities = new ArrayList<>(); + timescaleTsKvEntities.addAll(switchAggregation(key, startTs, startTs + interval, timeBucket, aggregation, entityId.getId())); + timescaleTsKvEntities.addAll(switchAggregation(key, startTs + interval, endTs, remainingPart, aggregation, entityId.getId())); + } + + if (!CollectionUtils.isEmpty(timescaleTsKvEntities)) { + List> result = new ArrayList<>(); + timescaleTsKvEntities.forEach(entity -> { + if (entity != null && entity.isNotEmpty()) { + entity.setEntityId(entityId.getId()); + entity.setStrKey(key); + result.add(Optional.of(entity)); + } else { + result.add(Optional.empty()); + } + }); + return result; + } else { + return Collections.emptyList(); + } } - private CompletableFuture> switchAggregation(String key, long startTs, long endTs, long timeBucket, Aggregation aggregation, UUID entityId) { + private List switchAggregation(String key, long startTs, long endTs, long timeBucket, Aggregation aggregation, UUID entityId) { + Integer keyId = getOrSaveKeyId(key); switch (aggregation) { case AVG: - return findAvg(key, startTs, endTs, timeBucket, entityId); + return aggregationRepository.findAvg(entityId, keyId, timeBucket, startTs, endTs); case MAX: - return findMax(key, startTs, endTs, timeBucket, entityId); + return aggregationRepository.findMax(entityId, keyId, timeBucket, startTs, endTs); case MIN: - return findMin(key, startTs, endTs, timeBucket, entityId); + return aggregationRepository.findMin(entityId, keyId, timeBucket, startTs, endTs); case SUM: - return findSum(key, startTs, endTs, timeBucket, entityId); + return aggregationRepository.findSum(entityId, keyId, timeBucket, startTs, endTs); case COUNT: - return findCount(key, startTs, endTs, timeBucket, entityId); + return aggregationRepository.findCount(entityId, keyId, timeBucket, startTs, endTs); default: throw new IllegalArgumentException("Not supported aggregation type: " + aggregation); } } - private CompletableFuture> findCount(String key, long startTs, long endTs, long timeBucket, UUID entityId) { - Integer keyId = getOrSaveKeyId(key); - return aggregationRepository.findCount( - entityId, - keyId, - timeBucket, - startTs, - endTs); - } - - private CompletableFuture> findSum(String key, long startTs, long endTs, long timeBucket, UUID entityId) { - Integer keyId = getOrSaveKeyId(key); - return aggregationRepository.findSum( - entityId, - keyId, - timeBucket, - startTs, - endTs); - } - - private CompletableFuture> findMin(String key, long startTs, long endTs, long timeBucket, UUID entityId) { - Integer keyId = getOrSaveKeyId(key); - return aggregationRepository.findMin( - entityId, - keyId, - timeBucket, - startTs, - endTs); - } - - private CompletableFuture> findMax(String key, long startTs, long endTs, long timeBucket, UUID entityId) { - Integer keyId = getOrSaveKeyId(key); - return aggregationRepository.findMax( - entityId, - keyId, - timeBucket, - startTs, - endTs); - } - - private CompletableFuture> findAvg(String key, long startTs, long endTs, long timeBucket, UUID entityId) { - Integer keyId = getOrSaveKeyId(key); - return aggregationRepository.findAvg( - entityId, - keyId, - timeBucket, - startTs, - endTs); - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java index c603d9d92b..1d9817a5bd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java @@ -20,14 +20,12 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; -import org.springframework.scheduling.annotation.Async; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sqlts.ts.TsKvCompositeKey; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import java.util.List; import java.util.UUID; -import java.util.concurrent.CompletableFuture; public interface TsKvRepository extends JpaRepository { @@ -48,82 +46,75 @@ public interface TsKvRepository extends JpaRepository= :startTs AND tskv.ts < :endTs") - CompletableFuture findStringMax(@Param("entityId") UUID entityId, + TsKvEntity findStringMax(@Param("entityId") UUID entityId, @Param("entityKey") int entityKey, @Param("startTs") long startTs, @Param("endTs") long endTs); - @Async @Query("SELECT new TsKvEntity(MAX(COALESCE(tskv.longValue, -9223372036854775807)), " + "MAX(COALESCE(tskv.doubleValue, -1.79769E+308)), " + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END), " + - "'MAX') FROM TsKvEntity tskv " + + "'MAX', MAX(tskv.ts)) FROM TsKvEntity tskv " + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts >= :startTs AND tskv.ts < :endTs") - CompletableFuture findNumericMax(@Param("entityId") UUID entityId, + TsKvEntity findNumericMax(@Param("entityId") UUID entityId, @Param("entityKey") int entityKey, @Param("startTs") long startTs, @Param("endTs") long endTs); - @Async - @Query("SELECT new TsKvEntity(MIN(tskv.strValue)) FROM TsKvEntity tskv " + + @Query("SELECT new TsKvEntity(MIN(tskv.strValue), MAX(tskv.ts)) FROM TsKvEntity tskv " + "WHERE tskv.strValue IS NOT NULL " + "AND tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts >= :startTs AND tskv.ts < :endTs") - CompletableFuture findStringMin(@Param("entityId") UUID entityId, + TsKvEntity findStringMin(@Param("entityId") UUID entityId, @Param("entityKey") int entityKey, @Param("startTs") long startTs, @Param("endTs") long endTs); - @Async @Query("SELECT new TsKvEntity(MIN(COALESCE(tskv.longValue, 9223372036854775807)), " + "MIN(COALESCE(tskv.doubleValue, 1.79769E+308)), " + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END), " + - "'MIN') FROM TsKvEntity tskv " + + "'MIN', MAX(tskv.ts)) FROM TsKvEntity tskv " + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts >= :startTs AND tskv.ts < :endTs") - CompletableFuture findNumericMin( + TsKvEntity findNumericMin( @Param("entityId") UUID entityId, @Param("entityKey") int entityKey, @Param("startTs") long startTs, @Param("endTs") long endTs); - @Async @Query("SELECT new TsKvEntity(SUM(CASE WHEN tskv.booleanValue IS NULL THEN 0 ELSE 1 END), " + "SUM(CASE WHEN tskv.strValue IS NULL THEN 0 ELSE 1 END), " + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END), " + - "SUM(CASE WHEN tskv.jsonValue IS NULL THEN 0 ELSE 1 END)) FROM TsKvEntity tskv " + + "SUM(CASE WHEN tskv.jsonValue IS NULL THEN 0 ELSE 1 END), MAX(tskv.ts)) FROM TsKvEntity tskv " + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts >= :startTs AND tskv.ts < :endTs") - CompletableFuture findCount(@Param("entityId") UUID entityId, + TsKvEntity findCount(@Param("entityId") UUID entityId, @Param("entityKey") int entityKey, @Param("startTs") long startTs, @Param("endTs") long endTs); - @Async @Query("SELECT new TsKvEntity(SUM(COALESCE(tskv.longValue, 0)), " + "SUM(COALESCE(tskv.doubleValue, 0.0)), " + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END), " + - "'AVG') FROM TsKvEntity tskv " + + "'AVG', MAX(tskv.ts)) FROM TsKvEntity tskv " + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts >= :startTs AND tskv.ts < :endTs") - CompletableFuture findAvg(@Param("entityId") UUID entityId, + TsKvEntity findAvg(@Param("entityId") UUID entityId, @Param("entityKey") int entityKey, @Param("startTs") long startTs, @Param("endTs") long endTs); - @Async @Query("SELECT new TsKvEntity(SUM(COALESCE(tskv.longValue, 0)), " + "SUM(COALESCE(tskv.doubleValue, 0.0)), " + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END), " + - "'SUM') FROM TsKvEntity tskv " + + "'SUM', MAX(tskv.ts)) FROM TsKvEntity tskv " + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts >= :startTs AND tskv.ts < :endTs") - CompletableFuture findSum(@Param("entityId") UUID entityId, + TsKvEntity findSum(@Param("entityId") UUID entityId, @Param("entityKey") int entityKey, @Param("startTs") long startTs, @Param("endTs") long endTs); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java index 975c132011..e774ad489a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java @@ -19,6 +19,7 @@ import com.datastax.oss.driver.api.core.cql.Row; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.kv.AggTsKvEntry; import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; @@ -28,6 +29,7 @@ import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntryAggWrapper; import org.thingsboard.server.dao.nosql.TbResultSet; import javax.annotation.Nullable; @@ -40,18 +42,20 @@ import java.util.stream.Collectors; * Created by ashvayka on 20.02.17. */ @Slf4j -public class AggregatePartitionsFunction implements com.google.common.util.concurrent.AsyncFunction, Optional> { +public class AggregatePartitionsFunction implements com.google.common.util.concurrent.AsyncFunction, Optional> { private static final int LONG_CNT_POS = 0; private static final int DOUBLE_CNT_POS = 1; private static final int BOOL_CNT_POS = 2; private static final int STR_CNT_POS = 3; private static final int JSON_CNT_POS = 4; - private static final int LONG_POS = 5; - private static final int DOUBLE_POS = 6; - private static final int BOOL_POS = 7; - private static final int STR_POS = 8; - private static final int JSON_POS = 9; + private static final int MAX_TS_POS = 5; + private static final int LONG_POS = 6; + private static final int DOUBLE_POS = 7; + private static final int BOOL_POS = 8; + private static final int STR_POS = 9; + private static final int JSON_POS = 10; + private final Aggregation aggregation; private final String key; @@ -66,29 +70,29 @@ public class AggregatePartitionsFunction implements com.google.common.util.concu } @Override - public ListenableFuture> apply(@Nullable List rsList) { - log.trace("[{}][{}][{}] Going to aggregate data", key, ts, aggregation); - if (rsList == null || rsList.isEmpty()) { - return Futures.immediateFuture(Optional.empty()); - } - return Futures.transform( - Futures.allAsList( - rsList.stream().map(rs -> rs.allRows(this.executor)) - .collect(Collectors.toList())), - rowsList -> { - try { - AggregationResult aggResult = new AggregationResult(); - for (List rs : rowsList) { - for (Row row : rs) { - processResultSetRow(row, aggResult); + public ListenableFuture> apply(@Nullable List rsList) { + log.trace("[{}][{}][{}] Going to aggregate data", key, ts, aggregation); + if (rsList == null || rsList.isEmpty()) { + return Futures.immediateFuture(Optional.empty()); + } + return Futures.transform( + Futures.allAsList( + rsList.stream().map(rs -> rs.allRows(this.executor)) + .collect(Collectors.toList())), + rowsList -> { + try { + AggregationResult aggResult = new AggregationResult(); + for (List rs : rowsList) { + for (Row row : rs) { + processResultSetRow(row, aggResult); + } + } + return processAggregationResult(aggResult); + } catch (Exception e) { + log.error("[{}][{}][{}] Failed to aggregate data", key, ts, aggregation, e); + return Optional.empty(); } - } - return processAggregationResult(aggResult); - } catch (Exception e) { - log.error("[{}][{}][{}] Failed to aggregate data", key, ts, aggregation, e); - return Optional.empty(); - } - }, this.executor); + }, this.executor); } private void processResultSetRow(Row row, AggregationResult aggResult) { @@ -105,6 +109,7 @@ public class AggregatePartitionsFunction implements com.google.common.util.concu long boolCount = row.getLong(BOOL_CNT_POS); long strCount = row.getLong(STR_CNT_POS); long jsonCount = row.getLong(JSON_CNT_POS); + long aggValuesLastTs = row.getLong(MAX_TS_POS); if (longCount > 0 || doubleCount > 0) { if (longCount > 0) { @@ -134,6 +139,8 @@ public class AggregatePartitionsFunction implements com.google.common.util.concu return; } + aggResult.aggValuesLastTs = Math.max(aggResult.aggValuesLastTs, aggValuesLastTs); + if (aggregation == Aggregation.COUNT) { aggResult.count += curCount; } else if (aggregation == Aggregation.AVG || aggregation == Aggregation.SUM) { @@ -231,34 +238,37 @@ public class AggregatePartitionsFunction implements com.google.common.util.concu } } - private Optional processAggregationResult(AggregationResult aggResult) { + private Optional processAggregationResult(AggregationResult aggResult) { Optional result; if (aggResult.dataType == null) { result = Optional.empty(); } else if (aggregation == Aggregation.COUNT) { result = Optional.of(new BasicTsKvEntry(ts, new LongDataEntry(key, aggResult.count))); } else if (aggregation == Aggregation.AVG || aggregation == Aggregation.SUM) { - result = processAvgOrSumResult(aggResult); + result = processAvgOrSumResult(aggregation, aggResult); } else if (aggregation == Aggregation.MIN || aggregation == Aggregation.MAX) { result = processMinOrMaxResult(aggResult); } else { result = Optional.empty(); } - if (!result.isPresent()) { + if (result.isEmpty()) { log.trace("[{}][{}][{}] Aggregated data is empty.", key, ts, aggregation); } - return result; + return result.map(tsKvEntry -> new TsKvEntryAggWrapper(tsKvEntry, aggResult.aggValuesLastTs)); } - private Optional processAvgOrSumResult(AggregationResult aggResult) { + private Optional processAvgOrSumResult(Aggregation aggregation, AggregationResult aggResult) { if (aggResult.count == 0 || (aggResult.dataType == DataType.DOUBLE && aggResult.dValue == null) || (aggResult.dataType == DataType.LONG && aggResult.lValue == null)) { return Optional.empty(); } else if (aggResult.dataType == DataType.DOUBLE || aggResult.dataType == DataType.LONG) { if (aggregation == Aggregation.AVG || aggResult.hasDouble) { double sum = Optional.ofNullable(aggResult.dValue).orElse(0.0d) + Optional.ofNullable(aggResult.lValue).orElse(0L); - return Optional.of(new BasicTsKvEntry(ts, new DoubleDataEntry(key, aggregation == Aggregation.SUM ? sum : (sum / aggResult.count)))); + DoubleDataEntry doubleDataEntry = new DoubleDataEntry(key, aggregation == Aggregation.SUM ? sum : (sum / aggResult.count)); + TsKvEntry result = aggregation == Aggregation.AVG ? new AggTsKvEntry(ts, doubleDataEntry, aggResult.count) : new BasicTsKvEntry(ts, doubleDataEntry); + return Optional.of(result); } else { - return Optional.of(new BasicTsKvEntry(ts, new LongDataEntry(key, aggregation == Aggregation.SUM ? aggResult.lValue : (aggResult.lValue / aggResult.count)))); + LongDataEntry longDataEntry = new LongDataEntry(key, aggregation == Aggregation.SUM ? aggResult.lValue : (aggResult.lValue / aggResult.count)); + return Optional.of(new BasicTsKvEntry(ts, longDataEntry)); } } return Optional.empty(); @@ -291,5 +301,6 @@ public class AggregatePartitionsFunction implements com.google.common.util.concu Long lValue = null; long count = 0; boolean hasDouble = false; + long aggValuesLastTs = 0; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index 8254b2bee0..b4cb27cb53 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.dao.entityview.EntityViewService; @@ -45,6 +46,7 @@ import org.thingsboard.server.dao.service.Validator; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.StringUtils.isBlank; @@ -52,6 +54,7 @@ import static org.thingsboard.server.common.data.StringUtils.isBlank; /** * @author Andrew Shvayka */ +@SuppressWarnings("UnstableApiUsage") @Service @Slf4j public class BaseTimeseriesService implements TimeseriesService { @@ -59,7 +62,7 @@ public class BaseTimeseriesService implements TimeseriesService { private static final int INSERTS_PER_ENTRY = 3; private static final int INSERTS_PER_ENTRY_WITHOUT_LATEST = 2; private static final int DELETES_PER_ENTRY = INSERTS_PER_ENTRY; - public static final Function, Integer> SUM_ALL_INTEGERS = new Function, Integer>() { + public static final Function, Integer> SUM_ALL_INTEGERS = new Function<>() { @Override public @Nullable Integer apply(@Nullable List input) { int result = 0; @@ -87,7 +90,7 @@ public class BaseTimeseriesService implements TimeseriesService { private EntityViewService entityViewService; @Override - public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, List queries) { + public ListenableFuture> findAllByQueries(TenantId tenantId, EntityId entityId, List queries) { validate(entityId); queries.forEach(this::validate); if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) { @@ -103,6 +106,17 @@ public class BaseTimeseriesService implements TimeseriesService { return timeseriesDao.findAllAsync(tenantId, entityId, queries); } + @Override + public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, List queries) { + return Futures.transform(findAllByQueries(tenantId, entityId, queries), + result -> { + if (result != null && !result.isEmpty()) { + return result.stream().map(ReadTsKvQueryResult::getData).flatMap(Collection::stream).collect(Collectors.toList()); + } + return Collections.emptyList(); + }, MoreExecutors.directExecutor()); + } + @Override public ListenableFuture> findLatest(TenantId tenantId, EntityId entityId, Collection keys) { validate(entityId); @@ -244,7 +258,7 @@ public class BaseTimeseriesService implements TimeseriesService { public ListenableFuture> removeAllLatest(TenantId tenantId, EntityId entityId) { validate(entityId); return Futures.transformAsync(this.findAllLatest(tenantId, entityId), latest -> { - if (!latest.isEmpty()) { + if (latest != null && !latest.isEmpty()) { Collection keys = latest.stream().map(TsKvEntry::getKey).collect(Collectors.toList()); return Futures.transform(this.removeLatest(tenantId, entityId, keys), res -> keys, MoreExecutors.directExecutor()); } else { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 3b5e45f206..367ad942a7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -42,7 +42,9 @@ import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntryAggWrapper; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.nosql.TbResultSetFuture; @@ -71,6 +73,7 @@ import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; /** * @author Andrew Shvayka */ +@SuppressWarnings("UnstableApiUsage") @Component @Slf4j @NoSqlTsDao @@ -139,20 +142,10 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } @Override - public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { - List>> futures = queries.stream().map(query -> findAllAsync(tenantId, entityId, query)).collect(Collectors.toList()); - return Futures.transform(Futures.allAsList(futures), new Function<>() { - @Nullable - @Override - public List apply(@Nullable List> results) { - if (results == null || results.isEmpty()) { - return null; - } - return results.stream() - .flatMap(List::stream) - .collect(Collectors.toList()); - } - }, readResultsProcessingExecutor); + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { + List> futures = queries.stream() + .map(query -> findAllAsync(tenantId, entityId, query)).collect(Collectors.toList()); + return Futures.allAsList(futures); } @Override @@ -270,28 +263,42 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } @Override - public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + public ListenableFuture findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { if (query.getAggregation() == Aggregation.NONE) { return findAllAsyncWithLimit(tenantId, entityId, query); } else { long startPeriod = query.getStartTs(); - long endPeriod = query.getEndTs(); + long endPeriod = Math.max(query.getStartTs() + 1, query.getEndTs()); long step = Math.max(query.getInterval(), MIN_AGGREGATION_STEP_MS); - List>> futures = new ArrayList<>(); - while (startPeriod <= endPeriod) { + List>> futures = new ArrayList<>(); + while (startPeriod < endPeriod) { long startTs = startPeriod; - long endTs = Math.min(startPeriod + step, endPeriod + 1); + long endTs = Math.min(startPeriod + step, endPeriod); long ts = endTs - startTs; ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); futures.add(findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs))); startPeriod = endTs; } - ListenableFuture>> future = Futures.allAsList(futures); + ListenableFuture>> future = Futures.allAsList(futures); return Futures.transform(future, new Function<>() { @Nullable @Override - public List apply(@Nullable List> input) { - return input == null ? Collections.emptyList() : input.stream().filter(v -> v.isPresent()).map(v -> v.get()).collect(Collectors.toList()); + public ReadTsKvQueryResult apply(@Nullable List> input) { + if (input == null) { + return new ReadTsKvQueryResult(query.getId(), Collections.emptyList(), query.getStartTs()); + } else { + long maxTs = query.getStartTs(); + List data = new ArrayList<>(); + for (var opt : input) { + if (opt.isPresent()) { + TsKvEntryAggWrapper tsKvEntryAggWrapper = opt.get(); + maxTs = Math.max(maxTs, tsKvEntryAggWrapper.getLastEntryTs()); + data.add(tsKvEntryAggWrapper.getEntry()); + } + } + return new ReadTsKvQueryResult(query.getId(), data, maxTs); + } + } }, readResultsProcessingExecutor); } @@ -302,13 +309,13 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD //Cleanup by TTL is native for Cassandra } - private ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + private ListenableFuture findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { long minPartition = toPartitionTs(query.getStartTs()); long maxPartition = toPartitionTs(query.getEndTs()); final ListenableFuture> partitionsListFuture = getPartitionsFuture(tenantId, query, entityId, minPartition, maxPartition); final SimpleListenableFuture> resultFuture = new SimpleListenableFuture<>(); - Futures.addCallback(partitionsListFuture, new FutureCallback>() { + Futures.addCallback(partitionsListFuture, new FutureCallback<>() { @Override public void onSuccess(@Nullable List partitions) { TsKvQueryCursor cursor = new TsKvQueryCursor(entityId.getEntityType().name(), entityId.getId(), query, partitions); @@ -321,7 +328,13 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } }, readResultsProcessingExecutor); - return resultFuture; + return Futures.transform(resultFuture, tsKvEntries -> { + long lastTs = query.getStartTs(); + if (tsKvEntries != null) { + lastTs = tsKvEntries.stream().map(TsKvEntry::getTs).max(Long::compare).orElse(query.getStartTs()); + } + return new ReadTsKvQueryResult(query.getId(), tsKvEntries, lastTs); + }, MoreExecutors.directExecutor()); } private long toPartitionTs(long ts) { @@ -379,7 +392,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } } - private ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long minPartition, long maxPartition) { + private ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long minPartition, long maxPartition) { final Aggregation aggregation = query.getAggregation(); final String key = query.getKey(); final long startTs = query.getStartTs(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index 124af03a1a..2b3d62712f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.dao.model.ModelConstants; @@ -145,9 +146,10 @@ public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimes long endTs = query.getStartTs() - 1; ReadTsKvQuery findNewLatestQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, endTs - startTs, 1, Aggregation.NONE, DESC_ORDER); - ListenableFuture> future = aggregationTimeseriesDao.findAllAsync(tenantId, entityId, findNewLatestQuery); + ListenableFuture future = aggregationTimeseriesDao.findAllAsync(tenantId, entityId, findNewLatestQuery); - return Futures.transformAsync(future, entryList -> { + return Futures.transformAsync(future, result -> { + var entryList = result.getData(); if (entryList.size() == 1) { TsKvEntry entry = entryList.get(0); return Futures.transform(saveLatest(tenantId, entityId, entryList.get(0)), v -> new TsKvLatestRemovingResult(entry), MoreExecutors.directExecutor()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java index c075a51434..5fd26d400a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java @@ -20,16 +20,18 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; +import java.util.Map; /** * @author Andrew Shvayka */ public interface TimeseriesDao { - ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries); + ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries); ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl); diff --git a/dao/src/test/java/org/thingsboard/server/dao/TimescaleDaoServiceTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/TimescaleDaoServiceTestSuite.java new file mode 100644 index 0000000000..6c29aea9ee --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/TimescaleDaoServiceTestSuite.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.junit.extensions.cpsuite.ClasspathSuite; +import org.junit.extensions.cpsuite.ClasspathSuite.ClassnameFilters; +import org.junit.runner.RunWith; + +@RunWith(ClasspathSuite.class) +@ClassnameFilters({ + "org.thingsboard.server.dao.service.*.nosql.*ServiceTimescaleTest", +}) +public class TimescaleDaoServiceTestSuite { + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/TimescaleSqlInitializer.java b/dao/src/test/java/org/thingsboard/server/dao/TimescaleSqlInitializer.java new file mode 100644 index 0000000000..991f959937 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/TimescaleSqlInitializer.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import com.google.common.base.Charsets; +import com.google.common.io.Resources; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.net.URL; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; + +@Slf4j +public class TimescaleSqlInitializer { + + private static final List sqlFiles = List.of( + "sql/schema-timescale.sql", + "sql/schema-entities.sql", + "sql/schema-entities-idx.sql", + "sql/schema-entities-idx-psql-addon.sql", + "sql/system-data.sql", + "sql/system-test-psql.sql"); + private static final String dropAllTablesSqlFile = "sql/timescale/drop-all-tables.sql"; + + public static void initDb(Connection conn) { + cleanUpDb(conn); + log.info("initialize Timescale DB..."); + try { + for (String sqlFile : sqlFiles) { + URL sqlFileUrl = Resources.getResource(sqlFile); + String sql = Resources.toString(sqlFileUrl, Charsets.UTF_8); + conn.createStatement().execute(sql); + } + } catch (IOException | SQLException e) { + throw new RuntimeException("Unable to init the Timescale database. Reason: " + e.getMessage(), e); + } + log.info("Timescale DB is initialized!"); + } + + private static void cleanUpDb(Connection conn) { + log.info("clean up Timescale DB..."); + try { + URL dropAllTableSqlFileUrl = Resources.getResource(dropAllTablesSqlFile); + String dropAllTablesSql = Resources.toString(dropAllTableSqlFileUrl, Charsets.UTF_8); + conn.createStatement().execute(dropAllTablesSql); + } catch (IOException | SQLException e) { + throw new RuntimeException("Unable to clean up the Timescale database. Reason: " + e.getMessage(), e); + } + } +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DaoTimescaleTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DaoTimescaleTest.java new file mode 100644 index 0000000000..237f539cf4 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DaoTimescaleTest.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service; + +import org.springframework.test.context.TestPropertySource; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Inherited +@Documented +@TestPropertySource(locations = {"classpath:application-test.properties", "classpath:timescale-test.properties"}) +public @interface DaoTimescaleTest { +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 5339863044..4e2fd17846 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -213,17 +213,17 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { } saveEntries(deviceId, TS + 100L + 1L); - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 100, 101, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 100, 100, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(1, entries.size()); - Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 10L)), entries.get(0)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(1, entries.size()); - Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 10L)), entries.get(0)); } @Test @@ -240,14 +240,14 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 75000 - 1, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 75000 - 1, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); } @Test @@ -264,14 +264,14 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); } @Test @@ -286,14 +286,14 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 75000 - 1, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 75000 - 1, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); } @Test diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/nosql/TimeseriesServiceTimescaleTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/nosql/TimeseriesServiceTimescaleTest.java new file mode 100644 index 0000000000..c36934cf15 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/nosql/TimeseriesServiceTimescaleTest.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.timeseries.nosql; + +import org.thingsboard.server.dao.service.DaoTimescaleTest; +import org.thingsboard.server.dao.service.timeseries.BaseTimeseriesServiceTest; + +@DaoTimescaleTest +public class TimeseriesServiceTimescaleTest extends BaseTimeseriesServiceTest { +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index db216f7f6e..9c4b1cbbea 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -19,8 +19,11 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.Optional; @@ -43,25 +46,25 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { final int LIMIT = 1; final String TEMP = "temp"; final String DESC = "DESC"; - AbstractChunkedAggregationTimeseriesDao tsDao; + private AbstractChunkedAggregationTimeseriesDao tsDao; @Before public void setUp() throws Exception { tsDao = spy(AbstractChunkedAggregationTimeseriesDao.class); - ListenableFuture> optionalListenableFuture = Futures.immediateFuture(Optional.of(mock(TsKvEntry.class))); - willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - willReturn(Futures.immediateFuture(mock(TsKvEntry.class))).given(tsDao).getTskvEntriesFuture(any()); + Optional optionalListenableFuture = Optional.of(mock(TsKvEntry.class)); + willReturn(Futures.immediateFuture(optionalListenableFuture)).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + willReturn(Futures.immediateFuture(mock(ReadTsKvQueryResult.class))).given(tsDao).getReadTsKvQueryResultFuture(any(), any()); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenLastIntervalShorterThanOthersAndEqualsEndTs() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2000, LIMIT, COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2001, 1001, LIMIT, COUNT, DESC); - ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3001, 2501, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3000, 2501, LIMIT, COUNT, DESC); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 2001, getTsForReadTsKvQuery(1, 2001), COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 2001, 3000 + 1, getTsForReadTsKvQuery(2001, 3001), COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 2001, 3000, getTsForReadTsKvQuery(2001, 3000), COUNT); } @Test @@ -71,19 +74,17 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); assertThat(tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query)).isNotNull(); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000 + 1, getTsForReadTsKvQuery(1, 3001), COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000, getTsForReadTsKvQuery(1, 3000), COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodMinusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2999, LIMIT, COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1500, LIMIT, COUNT, DESC); - ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 3000, 3001, 3000, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000, getTsForReadTsKvQuery(1, 3000), COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 3000, 3001, getTsForReadTsKvQuery(3000, 3001), COUNT); } @@ -94,7 +95,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3001, getTsForReadTsKvQuery(1, 3001), COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000, getTsForReadTsKvQuery(1, 3000), COUNT); } @Test @@ -134,7 +135,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3001, getTsForReadTsKvQuery(1, 3001), COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000, getTsForReadTsKvQuery(1, 3000), COUNT); } @Test @@ -144,8 +145,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1000)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); for (long i = 1; i <= 3000; i += 3) { - ReadTsKvQuery querySub = new BaseReadTsKvQuery(TEMP, i, i + 3, i + (i + 3 - i) / 2, LIMIT, COUNT, DESC); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, querySub.getKey(), i, i + 3, getTsForReadTsKvQuery(i, i + 3), COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, TEMP, i, Math.min(i + 3, 3000), getTsForReadTsKvQuery(i, i + 3), COUNT); } } diff --git a/dao/src/test/resources/timescale-test.properties b/dao/src/test/resources/timescale-test.properties new file mode 100644 index 0000000000..2c5552cb75 --- /dev/null +++ b/dao/src/test/resources/timescale-test.properties @@ -0,0 +1,18 @@ +database.ts.type=timescale +database.ts_latest.type=timescale + +sql.ts_inserts_executor_type=fixed +sql.ts_inserts_fixed_thread_pool_size=10 + +spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true +spring.jpa.properties.hibernate.order_by.default_null_ordering=last +spring.jpa.properties.hibernate.jdbc.log.warnings=false + +spring.jpa.show-sql=false + +spring.jpa.hibernate.ddl-auto=none +spring.datasource.username=postgres +spring.datasource.password=postgres +spring.datasource.url=jdbc:tc:timescaledb:latest-pg12:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw&?TC_INITFUNCTION=org.thingsboard.server.dao.TimescaleSqlInitializer::initDb +spring.datasource.driverClassName=org.testcontainers.jdbc.ContainerDatabaseDriver +spring.datasource.hikari.maximumPoolSize = 50 diff --git a/pom.xml b/pom.xml index 5fbfe077d4..347874faef 100755 --- a/pom.xml +++ b/pom.xml @@ -133,7 +133,7 @@ 1.3.0 1.2.7 - 1.16.0 + 1.17.3 1.12 3.0.0 6.1.0.202203080745-r diff --git a/ui-ngx/src/app/core/api/data-aggregator.ts b/ui-ngx/src/app/core/api/data-aggregator.ts index 7485a4bc87..6e77a0a161 100644 --- a/ui-ngx/src/app/core/api/data-aggregator.ts +++ b/ui-ngx/src/app/core/api/data-aggregator.ts @@ -14,7 +14,10 @@ /// limitations under the License. /// -import { SubscriptionData, SubscriptionDataHolder } from '@app/shared/models/telemetry/telemetry.models'; +import { + AggKey, + IndexedSubscriptionData, +} from '@app/shared/models/telemetry/telemetry.models'; import { AggregationType, calculateIntervalComparisonEndTime, @@ -25,10 +28,10 @@ import { SubscriptionTimewindow } from '@shared/models/time/time.models'; import { UtilsService } from '@core/services/utils.service'; -import { deepClone, isNumber, isNumeric } from '@core/utils'; +import { deepClone, isDefinedAndNotNull, isNumber, isNumeric } from '@core/utils'; import Timeout = NodeJS.Timeout; -export declare type onAggregatedData = (data: SubscriptionData, detectChanges: boolean) => void; +export declare type onAggregatedData = (data: IndexedSubscriptionData, detectChanges: boolean) => void; interface AggData { count: number; @@ -67,12 +70,12 @@ class AggDataMap { } class AggregationMap { - aggMap: {[key: string]: AggDataMap} = {}; + aggMap: {[id: number]: AggDataMap} = {}; detectRangeChanged(): boolean { let changed = false; - for (const key of Object.keys(this.aggMap)) { - const aggDataMap = this.aggMap[key]; + for (const id of Object.keys(this.aggMap)) { + const aggDataMap = this.aggMap[id]; if (aggDataMap.rangeChanged) { changed = true; aggDataMap.rangeChanged = false; @@ -82,8 +85,8 @@ class AggregationMap { } clearRangeChangedFlags() { - for (const key of Object.keys(this.aggMap)) { - this.aggMap[key].rangeChanged = false; + for (const id of Object.keys(this.aggMap)) { + this.aggMap[id].rangeChanged = false; } } } @@ -93,7 +96,7 @@ declare type AggFunction = (aggData: AggData, value?: any) => void; const avg: AggFunction = (aggData: AggData, value?: any) => { aggData.count++; if (isNumber(value)) { - aggData.sum += value; + aggData.sum = aggData.aggValue * (aggData.count - 1) + value; aggData.aggValue = aggData.sum / aggData.count; } else { aggData.aggValue = value; @@ -135,9 +138,25 @@ const none: AggFunction = (aggData: AggData, value?: any) => { export class DataAggregator { - private dataBuffer: SubscriptionData = {}; - private data: SubscriptionData; - private readonly lastPrevKvPairData: {[key: string]: [number, any]}; + constructor(private onDataCb: onAggregatedData, + private tsKeys: AggKey[], + private isLatestDataAgg: boolean, + private subsTw: SubscriptionTimewindow, + private utils: UtilsService, + private ignoreDataUpdateOnIntervalTick: boolean) { + this.tsKeys.forEach((key) => { + if (!this.dataBuffer[key.id]) { + this.dataBuffer[key.id] = []; + } + }); + if (this.subsTw.aggregation.stateData) { + this.lastPrevKvPairData = {}; + } + } + + private dataBuffer: IndexedSubscriptionData = []; + private data: IndexedSubscriptionData; + private readonly lastPrevKvPairData: {[id: number]: [number, any]}; private aggregationMap: AggregationMap; @@ -145,9 +164,7 @@ export class DataAggregator { private resetPending = false; private updatedData = false; - private noAggregation = this.subsTw.aggregation.type === AggregationType.NONE; - private aggregationTimeout = Math.max(this.subsTw.aggregation.interval, 1000); - private readonly aggFunction: AggFunction; + private aggregationTimeout = this.isLatestDataAgg ? 1000 : Math.max(this.subsTw.aggregation.interval, 1000); private intervalTimeoutHandle: Timeout; private intervalScheduledTime: number; @@ -156,38 +173,29 @@ export class DataAggregator { private endTs: number; private elapsed: number; - constructor(private onDataCb: onAggregatedData, - private tsKeyNames: string[], - private subsTw: SubscriptionTimewindow, - private utils: UtilsService, - private ignoreDataUpdateOnIntervalTick: boolean) { - this.tsKeyNames.forEach((key) => { - this.dataBuffer[key] = []; - }); - if (this.subsTw.aggregation.stateData) { - this.lastPrevKvPairData = {}; + private static convertValue(val: string, noAggregation: boolean): any { + if (val && isNumeric(val) && (!noAggregation || noAggregation && Number(val).toString() === val)) { + return Number(val); } - switch (this.subsTw.aggregation.type) { + return val; + } + + private static getAggFunction(aggType: AggregationType): AggFunction { + switch (aggType) { case AggregationType.MIN: - this.aggFunction = min; - break; + return min; case AggregationType.MAX: - this.aggFunction = max; - break; + return max; case AggregationType.AVG: - this.aggFunction = avg; - break; + return avg; case AggregationType.SUM: - this.aggFunction = sum; - break; + return sum; case AggregationType.COUNT: - this.aggFunction = count; - break; + return count; case AggregationType.NONE: - this.aggFunction = none; - break; + return none; default: - this.aggFunction = avg; + return avg; } } @@ -206,7 +214,7 @@ export class DataAggregator { this.intervalScheduledTime = this.utils.currentPerfTime(); this.calculateStartEndTs(); this.elapsed = 0; - this.aggregationTimeout = Math.max(this.subsTw.aggregation.interval, 1000); + this.aggregationTimeout = this.isLatestDataAgg ? 1000 : Math.max(this.subsTw.aggregation.interval, 1000); this.resetPending = true; this.updatedData = false; this.intervalTimeoutHandle = setTimeout(this.onInterval.bind(this), this.aggregationTimeout); @@ -220,7 +228,7 @@ export class DataAggregator { this.aggregationMap = null; } - public onData(data: SubscriptionDataHolder, update: boolean, history: boolean, detectChanges: boolean) { + public onData(data: IndexedSubscriptionData, update: boolean, history: boolean, detectChanges: boolean) { this.updatedData = true; if (!this.dataReceived || this.resetPending) { let updateIntervalScheduledTime = true; @@ -235,9 +243,9 @@ export class DataAggregator { } if (update) { this.aggregationMap = new AggregationMap(); - this.updateAggregatedData(data.data); + this.updateAggregatedData(data); } else { - this.aggregationMap = this.processAggregatedData(data.data); + this.aggregationMap = this.processAggregatedData(data); } if (updateIntervalScheduledTime) { this.intervalScheduledTime = this.utils.currentPerfTime(); @@ -245,7 +253,7 @@ export class DataAggregator { this.aggregationMap.clearRangeChangedFlags(); this.onInterval(history, detectChanges); } else { - this.updateAggregatedData(data.data); + this.updateAggregatedData(data); if (history) { this.intervalScheduledTime = this.utils.currentPerfTime(); this.onInterval(history, detectChanges); @@ -283,9 +291,9 @@ export class DataAggregator { } const intervalTimeout = rangeChanged ? this.aggregationTimeout - this.elapsed : this.aggregationTimeout; if (!history) { - const delta = Math.floor(this.elapsed / this.subsTw.aggregation.interval); + const delta = Math.floor(this.elapsed / this.aggregationTimeout); if (delta || !this.data || rangeChanged) { - const tickTs = delta * this.subsTw.aggregation.interval; + const tickTs = delta * this.aggregationTimeout; if (this.subsTw.quickInterval) { const startEndTime = calculateIntervalStartEndTime(this.subsTw.quickInterval, this.subsTw.timezone); this.startTs = startEndTime[0] + this.subsTw.tsOffset; @@ -295,7 +303,7 @@ export class DataAggregator { this.endTs += tickTs; } this.data = this.updateData(); - this.elapsed = this.elapsed - delta * this.subsTw.aggregation.interval; + this.elapsed = this.elapsed - delta * this.aggregationTimeout; } } else { this.data = this.updateData(); @@ -309,39 +317,45 @@ export class DataAggregator { } } - private updateData(): SubscriptionData { - this.tsKeyNames.forEach((key) => { - this.dataBuffer[key] = []; + private updateData(): IndexedSubscriptionData { + this.dataBuffer = []; + this.tsKeys.forEach((key) => { + if (!this.dataBuffer[key.id]) { + this.dataBuffer[key.id] = []; + } }); - for (const key of Object.keys(this.aggregationMap.aggMap)) { - const aggKeyData = this.aggregationMap.aggMap[key]; - let keyData = this.dataBuffer[key]; + for (const idStr of Object.keys(this.aggregationMap.aggMap)) { + const id = Number(idStr); + const aggKeyData = this.aggregationMap.aggMap[id]; + const aggKey = this.aggKeyById(id); + const noAggregation = aggKey.agg === AggregationType.NONE; + let keyData = this.dataBuffer[id]; aggKeyData.forEach((aggData, aggTimestamp) => { if (aggTimestamp < this.startTs) { if (this.subsTw.aggregation.stateData && - (!this.lastPrevKvPairData[key] || this.lastPrevKvPairData[key][0] < aggTimestamp)) { - this.lastPrevKvPairData[key] = [aggTimestamp, aggData.aggValue]; + (!this.lastPrevKvPairData[id] || this.lastPrevKvPairData[id][0] < aggTimestamp)) { + this.lastPrevKvPairData[id] = [aggTimestamp, aggData.aggValue]; } aggKeyData.delete(aggTimestamp); this.updatedData = true; - } else if (aggTimestamp < this.endTs || this.noAggregation) { + } else if (aggTimestamp < this.endTs || noAggregation) { const kvPair: [number, any] = [aggTimestamp, aggData.aggValue]; keyData.push(kvPair); } }); keyData.sort((set1, set2) => set1[0] - set2[0]); if (this.subsTw.aggregation.stateData) { - this.updateStateBounds(keyData, deepClone(this.lastPrevKvPairData[key])); + this.updateStateBounds(keyData, deepClone(this.lastPrevKvPairData[id])); } if (keyData.length > this.subsTw.aggregation.limit) { keyData = keyData.slice(keyData.length - this.subsTw.aggregation.limit); } - this.dataBuffer[key] = keyData; + this.dataBuffer[id] = keyData; } return this.dataBuffer; } - private updateStateBounds(keyData: [number, any][], lastPrevKvPair: [number, any]) { + private updateStateBounds(keyData: [number, any, number?][], lastPrevKvPair: [number, any]) { if (lastPrevKvPair) { lastPrevKvPair[0] = this.startTs; } @@ -369,66 +383,71 @@ export class DataAggregator { } } - private processAggregatedData(data: SubscriptionData): AggregationMap { - const isCount = this.subsTw.aggregation.type === AggregationType.COUNT; + private processAggregatedData(data: IndexedSubscriptionData): AggregationMap { const aggregationMap = new AggregationMap(); - for (const key of Object.keys(data)) { - let aggKeyData = aggregationMap.aggMap[key]; + for (const idStr of Object.keys(data)) { + const id = Number(idStr); + const aggKey = this.aggKeyById(id); + const aggType = aggKey.agg; + const isCount = aggType === AggregationType.COUNT; + const noAggregation = aggType === AggregationType.NONE; + let aggKeyData = aggregationMap.aggMap[id]; if (!aggKeyData) { aggKeyData = new AggDataMap(); - aggregationMap.aggMap[key] = aggKeyData; + aggregationMap.aggMap[id] = aggKeyData; } - const keyData = data[key]; + const keyData = data[id]; keyData.forEach((kvPair) => { const timestamp = kvPair[0]; - const value = this.convertValue(kvPair[1]); - const aggKey = timestamp; + const value = DataAggregator.convertValue(kvPair[1], noAggregation); + const tsKey = timestamp; const aggData = { - count: isCount ? value : 1, + count: isCount ? value : isDefinedAndNotNull(kvPair[2]) ? kvPair[2] : 1, sum: value, aggValue: value }; - aggKeyData.set(aggKey, aggData); + aggKeyData.set(tsKey, aggData); }); } return aggregationMap; } - private updateAggregatedData(data: SubscriptionData) { - const isCount = this.subsTw.aggregation.type === AggregationType.COUNT; - for (const key of Object.keys(data)) { - let aggKeyData = this.aggregationMap.aggMap[key]; + private updateAggregatedData(data: IndexedSubscriptionData) { + for (const idStr of Object.keys(data)) { + const id = Number(idStr); + const aggKey = this.aggKeyById(id); + const aggType = aggKey.agg; + const isCount = aggType === AggregationType.COUNT; + const noAggregation = aggType === AggregationType.NONE; + let aggKeyData = this.aggregationMap.aggMap[id]; if (!aggKeyData) { aggKeyData = new AggDataMap(); - this.aggregationMap.aggMap[key] = aggKeyData; + this.aggregationMap.aggMap[id] = aggKeyData; } - const keyData = data[key]; + const keyData = data[id]; keyData.forEach((kvPair) => { const timestamp = kvPair[0]; - const value = this.convertValue(kvPair[1]); - const aggTimestamp = this.noAggregation ? timestamp : (this.startTs + + const value = DataAggregator.convertValue(kvPair[1], noAggregation); + const aggTimestamp = noAggregation ? timestamp : (this.startTs + Math.floor((timestamp - this.startTs) / this.subsTw.aggregation.interval) * this.subsTw.aggregation.interval + this.subsTw.aggregation.interval / 2); let aggData = aggKeyData.get(aggTimestamp); if (!aggData) { aggData = { - count: 1, + count: isDefinedAndNotNull(kvPair[2]) ? kvPair[2] : 1, sum: value, aggValue: isCount ? 1 : value }; aggKeyData.set(aggTimestamp, aggData); } else { - this.aggFunction(aggData, value); + DataAggregator.getAggFunction(aggType)(aggData, value); } }); } } - private convertValue(val: string): any { - if (val && isNumeric(val) && (!this.noAggregation || this.noAggregation && Number(val).toString() === val)) { - return Number(val); - } - return val; + private aggKeyById(id: number): AggKey { + return this.tsKeys.find(key => key.id === id); } } diff --git a/ui-ngx/src/app/core/api/entity-data-subscription.ts b/ui-ngx/src/app/core/api/entity-data-subscription.ts index 9cffe57ae7..4dc0e7c4e1 100644 --- a/ui-ngx/src/app/core/api/entity-data-subscription.ts +++ b/ui-ngx/src/app/core/api/entity-data-subscription.ts @@ -14,9 +14,16 @@ /// limitations under the License. /// -import { DataSet, DataSetHolder, DatasourceType, widgetType } from '@shared/models/widget.models'; -import { AggregationType, getCurrentTime, SubscriptionTimewindow } from '@shared/models/time/time.models'; +import { ComparisonResultType, DataSet, DataSetHolder, DatasourceType, widgetType } from '@shared/models/widget.models'; import { + AggregationType, + ComparisonDuration, + createTimewindowForComparison, + getCurrentTime, + SubscriptionTimewindow +} from '@shared/models/time/time.models'; +import { + ComparisonTsValue, EntityData, EntityDataPageLink, EntityFilter, @@ -28,11 +35,12 @@ import { TsValue } from '@shared/models/query/query.models'; import { + AggKey, DataKeyType, EntityCountCmd, EntityDataCmd, + IndexedSubscriptionData, SubscriptionData, - SubscriptionDataHolder, TelemetryService, TelemetrySubscriber } from '@shared/models/telemetry/telemetry.models'; @@ -55,6 +63,11 @@ declare type DataUpdatedCb = (data: DataSetHolder, dataIndex: number, export interface SubscriptionDataKey { name: string; type: DataKeyType; + aggregationType?: AggregationType; + comparisonEnabled?: boolean; + timeForComparison?: ComparisonDuration; + comparisonCustomIntervalValue?: number; + comparisonResultType?: ComparisonResultType; funcBody: string; func?: DataKeyFunction; postFuncBody: string; @@ -82,9 +95,16 @@ export interface EntityDataSubscriptionOptions { export class EntityDataSubscription { + constructor(private listener: EntityDataListener, + private telemetryService: TelemetryService, + private utils: UtilsService) { + this.initializeSubscription(); + } + private entityDataSubscriptionOptions = this.listener.subscriptionOptions; private datasourceType: DatasourceType = this.entityDataSubscriptionOptions.datasourceType; private history: boolean; + private isFloatingTimewindow: boolean; private realtime: boolean; private subscriber: TelemetrySubscriber; @@ -95,13 +115,18 @@ export class EntityDataSubscription { private attrFields: Array; private tsFields: Array; private latestValues: Array; + private aggTsValues: Array; + private aggTsComparisonValues: Array; private entityDataResolveSubject: Subject; private pageData: PageData; + private data: Array>; private subsTw: SubscriptionTimewindow; private latestTsOffset: number; private dataAggregators: Array; + private tsLatestDataAggregators: Array; private dataKeys: {[key: string]: Array | SubscriptionDataKey} = {}; + private dataKeysList: SubscriptionDataKey[] = []; private datasourceData: {[index: number]: {[key: string]: DataSetHolder}}; private datasourceOrigData: {[index: number]: {[key: string]: DataSetHolder}}; private entityIdToDataIndex: {[id: string]: number}; @@ -116,15 +141,44 @@ export class EntityDataSubscription { private dataResolved = false; private started = false; - constructor(private listener: EntityDataListener, - private telemetryService: TelemetryService, - private utils: UtilsService) { - this.initializeSubscription(); + private static convertValue(val: string): any { + if (val && isNumeric(val) && Number(val).toString() === val) { + return Number(val); + } + return val; + } + + private static calculateComparisonValue(key: SubscriptionDataKey, comparisonTsValue: ComparisonTsValue): [number, any, number?][] { + let timestamp: number; + let value: any; + switch (key.comparisonResultType) { + case ComparisonResultType.PREVIOUS_VALUE: + timestamp = comparisonTsValue.previous.ts; + value = comparisonTsValue.previous.value; + break; + case ComparisonResultType.DELTA_ABSOLUTE: + case ComparisonResultType.DELTA_PERCENT: + timestamp = comparisonTsValue.previous.ts; + const currentVal = EntityDataSubscription.convertValue(comparisonTsValue.current.value); + const prevVal = EntityDataSubscription.convertValue(comparisonTsValue.previous.value); + if (isNumeric(currentVal) && isNumeric(prevVal)) { + if (key.comparisonResultType === ComparisonResultType.DELTA_ABSOLUTE) { + value = currentVal - prevVal; + } else { + value = (currentVal - prevVal) / prevVal * 100; + } + } else { + value = ''; + } + break; + } + return [[timestamp, value]]; } private initializeSubscription() { for (let i = 0; i < this.entityDataSubscriptionOptions.dataKeys.length; i++) { const dataKey = deepClone(this.entityDataSubscriptionOptions.dataKeys[i]); + this.dataKeysList.push(dataKey); dataKey.index = i; if (this.datasourceType === DatasourceType.function) { if (!dataKey.func) { @@ -142,7 +196,8 @@ export class EntityDataSubscription { if (this.datasourceType === DatasourceType.function) { key = `${dataKey.name}_${dataKey.index}_${dataKey.type}${dataKey.latest ? '_latest' : ''}`; } else { - key = `${dataKey.name}_${dataKey.type}${dataKey.latest ? '_latest' : ''}`; + const keyIndexSuffix = dataKey.aggregationType && dataKey.aggregationType !== AggregationType.NONE ? `_${dataKey.index}` : ''; + key = `${dataKey.name}_${dataKey.type}${keyIndexSuffix}${dataKey.latest ? '_latest' : ''}`; } let dataKeysList = this.dataKeys[key] as Array; if (!dataKeysList) { @@ -180,6 +235,12 @@ export class EntityDataSubscription { }); this.dataAggregators = null; } + if (this.tsLatestDataAggregators) { + this.tsLatestDataAggregators.forEach((aggregator) => { + aggregator.destroy(); + }); + this.tsLatestDataAggregators = null; + } this.pageData = null; } @@ -188,16 +249,11 @@ export class EntityDataSubscription { if (this.entityDataSubscriptionOptions.isPaginatedDataSubscription) { this.started = true; this.dataResolved = true; - this.subsTw = this.entityDataSubscriptionOptions.subscriptionTimewindow; - this.latestTsOffset = this.entityDataSubscriptionOptions.latestTsOffset; - this.history = this.entityDataSubscriptionOptions.subscriptionTimewindow && - isObject(this.entityDataSubscriptionOptions.subscriptionTimewindow.fixedWindow); - this.realtime = this.entityDataSubscriptionOptions.subscriptionTimewindow && - isDefinedAndNotNull(this.entityDataSubscriptionOptions.subscriptionTimewindow.realtimeWindowMs); + this.prepareSubscriptionTimewindow(); } if (this.datasourceType === DatasourceType.entity) { const entityFields: Array = - this.entityDataSubscriptionOptions.dataKeys.filter(dataKey => dataKey.type === DataKeyType.entityField).map( + this.dataKeysList.filter(dataKey => dataKey.type === DataKeyType.entityField).map( dataKey => ({ type: EntityKeyType.ENTITY_FIELD, key: dataKey.name }) ); if (!entityFields.find(key => key.key === 'name')) { @@ -219,18 +275,20 @@ export class EntityDataSubscription { }); } - this.attrFields = this.entityDataSubscriptionOptions.dataKeys.filter(dataKey => dataKey.type === DataKeyType.attribute).map( + this.attrFields = this.dataKeysList.filter(dataKey => dataKey.type === DataKeyType.attribute).map( dataKey => ({ type: EntityKeyType.ATTRIBUTE, key: dataKey.name }) ); - this.tsFields = this.entityDataSubscriptionOptions.dataKeys. - filter(dataKey => dataKey.type === DataKeyType.timeseries && !dataKey.latest).map( + this.tsFields = this.dataKeysList. + filter(dataKey => dataKey.type === DataKeyType.timeseries && + (!dataKey.aggregationType || dataKey.aggregationType === AggregationType.NONE) && !dataKey.latest).map( dataKey => ({ type: EntityKeyType.TIME_SERIES, key: dataKey.name }) ); if (this.entityDataSubscriptionOptions.type === widgetType.timeseries) { - const latestTsFields = this.entityDataSubscriptionOptions.dataKeys. - filter(dataKey => dataKey.type === DataKeyType.timeseries && dataKey.latest).map( + const latestTsFields = this.dataKeysList. + filter(dataKey => dataKey.type === DataKeyType.timeseries && dataKey.latest && + (!dataKey.aggregationType || dataKey.aggregationType === AggregationType.NONE)).map( dataKey => ({ type: EntityKeyType.TIME_SERIES, key: dataKey.name }) ); this.latestValues = this.attrFields.concat(latestTsFields); @@ -238,6 +296,19 @@ export class EntityDataSubscription { this.latestValues = this.attrFields.concat(this.tsFields); } + this.aggTsValues = this.dataKeysList. + filter(dataKey => dataKey.type === DataKeyType.timeseries && + dataKey.aggregationType && dataKey.aggregationType !== AggregationType.NONE && !dataKey.comparisonEnabled).map( + dataKey => ({ id: dataKey.index, key: dataKey.name, agg: dataKey.aggregationType }) + ); + + this.aggTsComparisonValues = this.dataKeysList. + filter(dataKey => dataKey.type === DataKeyType.timeseries && + dataKey.aggregationType && dataKey.aggregationType !== AggregationType.NONE && dataKey.comparisonEnabled).map( + dataKey => ({ id: dataKey.index, key: dataKey.name, agg: dataKey.aggregationType, + previousValueOnly: dataKey.comparisonResultType === ComparisonResultType.PREVIOUS_VALUE }) + ); + this.subscriber = new TelemetrySubscriber(this.telemetryService); this.dataCommand = new EntityDataCmd(); @@ -282,19 +353,28 @@ export class EntityDataSubscription { this.subscriber.reconnect$.subscribe(() => { if (this.started) { const targetCommand = this.entityDataSubscriptionOptions.isPaginatedDataSubscription ? this.dataCommand : this.subsCommand; - if (this.entityDataSubscriptionOptions.type === widgetType.timeseries && - !this.history && this.tsFields.length) { + if (!this.history && (this.entityDataSubscriptionOptions.type === widgetType.timeseries && this.tsFields.length || + this.aggTsValues.length > 0 && !this.isFloatingTimewindow)) { const newSubsTw = this.listener.updateRealtimeSubscription(); this.subsTw = newSubsTw; - targetCommand.tsCmd.startTs = this.subsTw.startTs; - targetCommand.tsCmd.timeWindow = this.subsTw.aggregation.timeWindow; - targetCommand.tsCmd.interval = this.subsTw.aggregation.interval; - targetCommand.tsCmd.limit = this.subsTw.aggregation.limit; - targetCommand.tsCmd.agg = this.subsTw.aggregation.type; - targetCommand.tsCmd.fetchLatestPreviousPoint = this.subsTw.aggregation.stateData; - this.dataAggregators.forEach((dataAggregator) => { - dataAggregator.reset(newSubsTw); - }); + if (this.entityDataSubscriptionOptions.type === widgetType.timeseries && this.tsFields.length) { + targetCommand.tsCmd.startTs = this.subsTw.startTs; + targetCommand.tsCmd.timeWindow = this.subsTw.aggregation.timeWindow; + targetCommand.tsCmd.interval = this.subsTw.aggregation.interval; + targetCommand.tsCmd.limit = this.subsTw.aggregation.limit; + targetCommand.tsCmd.agg = this.subsTw.aggregation.type; + targetCommand.tsCmd.fetchLatestPreviousPoint = this.subsTw.aggregation.stateData; + this.dataAggregators.forEach((dataAggregator) => { + dataAggregator.reset(newSubsTw); + }); + } + if (this.aggTsValues.length > 0 && !this.isFloatingTimewindow) { + targetCommand.aggTsCmd.startTs = this.subsTw.startTs; + targetCommand.aggTsCmd.timeWindow = this.subsTw.aggregation.timeWindow; + this.tsLatestDataAggregators.forEach((dataAggregator) => { + dataAggregator.reset(newSubsTw); + }); + } } if (this.entityDataSubscriptionOptions.type === widgetType.timeseries) { this.subscriber.setTsOffset(this.subsTw.tsOffset); @@ -359,7 +439,7 @@ export class EntityDataSubscription { entityType: null }; - const countKey = this.entityDataSubscriptionOptions.dataKeys[0]; + const countKey = this.dataKeysList[0]; let dataReceived = false; @@ -422,14 +502,9 @@ export class EntityDataSubscription { if (this.entityDataSubscriptionOptions.isPaginatedDataSubscription) { return; } - this.subsTw = this.entityDataSubscriptionOptions.subscriptionTimewindow; - this.latestTsOffset = this.entityDataSubscriptionOptions.latestTsOffset; - this.history = this.entityDataSubscriptionOptions.subscriptionTimewindow && - isObject(this.entityDataSubscriptionOptions.subscriptionTimewindow.fixedWindow); - this.realtime = this.entityDataSubscriptionOptions.subscriptionTimewindow && - isDefinedAndNotNull(this.entityDataSubscriptionOptions.subscriptionTimewindow.realtimeWindowMs); + this.prepareSubscriptionTimewindow(); - this.prepareData(); + this.prepareData(true); if (this.datasourceType === DatasourceType.entity) { this.subsCommand = new EntityDataCmd(); @@ -461,7 +536,19 @@ export class EntityDataSubscription { this.started = true; } + private prepareSubscriptionTimewindow() { + this.subsTw = this.entityDataSubscriptionOptions.subscriptionTimewindow; + this.latestTsOffset = this.entityDataSubscriptionOptions.latestTsOffset; + this.history = this.entityDataSubscriptionOptions.subscriptionTimewindow && + isObject(this.entityDataSubscriptionOptions.subscriptionTimewindow.fixedWindow); + this.realtime = this.entityDataSubscriptionOptions.subscriptionTimewindow && + isDefinedAndNotNull(this.entityDataSubscriptionOptions.subscriptionTimewindow.realtimeWindowMs); + this.isFloatingTimewindow = this.entityDataSubscriptionOptions.subscriptionTimewindow && + !this.entityDataSubscriptionOptions.subscriptionTimewindow.quickInterval && !this.history; + } + private prepareSubscriptionCommands(cmd: EntityDataCmd) { + let latestValuesKeys: EntityKey[] = []; if (this.entityDataSubscriptionOptions.type === widgetType.timeseries) { if (this.tsFields.length > 0) { if (this.history) { @@ -486,17 +573,40 @@ export class EntityDataSubscription { }; } } - if (this.latestValues.length > 0) { - cmd.latestCmd = { - keys: this.latestValues - }; - } + latestValuesKeys = this.latestValues; } else if (this.entityDataSubscriptionOptions.type === widgetType.latest) { - if (this.latestValues.length > 0) { - cmd.latestCmd = { - keys: this.latestValues - }; + latestValuesKeys = this.latestValues; + } + if (this.history && (this.aggTsValues.length > 0 || this.aggTsComparisonValues.length > 0)) { + for (const aggTsComparison of this.aggTsComparisonValues) { + const subscriptionDataKey = this.dataKeyByIndex(aggTsComparison.id); + const timewindowForComparison = + createTimewindowForComparison(this.subsTw, subscriptionDataKey.timeForComparison, + subscriptionDataKey.comparisonCustomIntervalValue); + aggTsComparison.previousStartTs = timewindowForComparison.fixedWindow.startTimeMs; + aggTsComparison.previousEndTs = timewindowForComparison.fixedWindow.endTimeMs; } + cmd.aggHistoryCmd = { + keys: [...this.aggTsValues, ...this.aggTsComparisonValues], + startTs: this.subsTw.fixedWindow.startTimeMs, + endTs: this.subsTw.fixedWindow.endTimeMs + }; + } else if (!this.isFloatingTimewindow && this.aggTsValues.length > 0) { + cmd.aggTsCmd = { + keys: this.aggTsValues, + startTs: this.subsTw.startTs, + timeWindow: this.subsTw.aggregation.timeWindow + }; + if (latestValuesKeys.length > 0) { + const tsKeys = this.aggTsValues.map(key => key.key); + latestValuesKeys = latestValuesKeys.filter(latestKey => latestKey.type !== EntityKeyType.TIME_SERIES + || !tsKeys.includes(latestKey.key)); + } + } + if (latestValuesKeys.length > 0) { + cmd.latestCmd = { + keys: latestValuesKeys + }; } } @@ -510,7 +620,7 @@ export class EntityDataSubscription { this.generateData(true); } - private prepareData() { + private prepareData(isUpdate: boolean) { if (this.timeseriesTimer) { clearTimeout(this.timeseriesTimer); this.timeseriesTimer = null; @@ -526,37 +636,73 @@ export class EntityDataSubscription { }); } this.dataAggregators = []; + if (this.tsLatestDataAggregators) { + this.tsLatestDataAggregators.forEach((aggregator) => { + aggregator.destroy(); + }); + } + this.tsLatestDataAggregators = []; this.resetData(); if (this.entityDataSubscriptionOptions.type === widgetType.timeseries) { - let tsKeyNames = []; + let tsKeyIds: number[]; if (this.datasourceType === DatasourceType.function) { - for (const key of Object.keys(this.dataKeys)) { - const dataKeysList = this.dataKeys[key] as Array; - dataKeysList.forEach((subscriptionDataKey) => { - if (!subscriptionDataKey.latest) { - tsKeyNames.push(`${subscriptionDataKey.name}_${subscriptionDataKey.index}`); - } - }); - } + tsKeyIds = this.dataKeysList.filter(key => !key.latest).map(key => key.index); } else { - tsKeyNames = this.tsFields ? this.tsFields.map(field => field.key) : []; + tsKeyIds = this.dataKeysList. + filter(dataKey => dataKey.type === DataKeyType.timeseries && + (!dataKey.aggregationType || dataKey.aggregationType === AggregationType.NONE) && !dataKey.latest).map( + dataKey => dataKey.index + ); } - for (let dataIndex = 0; dataIndex < this.pageData.data.length; dataIndex++) { - if (tsKeyNames.length) { - if (this.datasourceType === DatasourceType.function) { - this.dataAggregators[dataIndex] = this.createRealtimeDataAggregator(this.subsTw, tsKeyNames, - DataKeyType.function, dataIndex, this.notifyListener.bind(this)); - } else { - this.dataAggregators[dataIndex] = this.createRealtimeDataAggregator(this.subsTw, tsKeyNames, - DataKeyType.timeseries, dataIndex, this.notifyListener.bind(this)); - } + const aggKeys: AggKey[] = tsKeyIds.map(key => ({id: key, key: key + '', agg: this.subsTw.aggregation.type})); + if (aggKeys.length) { + for (let dataIndex = 0; dataIndex < this.pageData.data.length; dataIndex++) { + this.dataAggregators[dataIndex] = this.createRealtimeDataAggregator(this.subsTw, aggKeys, + false, dataIndex, this.notifyListener.bind(this)); } } } + if (this.aggTsValues && this.aggTsValues.length) { + if (!this.isFloatingTimewindow) { + const aggLatestTimewindow = deepClone(this.subsTw); + aggLatestTimewindow.aggregation.stateData = false; + aggLatestTimewindow.aggregation.interval = aggLatestTimewindow.aggregation.timeWindow; + for (let dataIndex = 0; dataIndex < this.pageData.data.length; dataIndex++) { + this.tsLatestDataAggregators[dataIndex] = this.createRealtimeDataAggregator(aggLatestTimewindow, this.aggTsValues, + true, dataIndex, this.notifyListener.bind(this)); + } + } else { + this.reportNotSupported(this.aggTsValues, isUpdate); + } + } + if (!this.history && this.aggTsComparisonValues && this.aggTsComparisonValues.length) { + this.reportNotSupported(this.aggTsComparisonValues, isUpdate); + } + } + + private reportNotSupported(keys: AggKey[], isUpdate: boolean) { + const indexedData: IndexedSubscriptionData = []; + for (const key of keys) { + indexedData[key.id] = [[0, 'Not supported!']]; + } + for (let dataIndex = 0; dataIndex < this.pageData.data.length; dataIndex++) { + this.onIndexedData(indexedData, dataIndex, true, + this.entityDataSubscriptionOptions.type === widgetType.timeseries, + (data, dataIndex1, dataKeyIndex, detectChanges, isLatest) => { + if (!this.data[dataIndex1]) { + this.data[dataIndex1] = []; + } + this.data[dataIndex1][dataKeyIndex] = data; + if (isUpdate) { + this.notifyListener(data, dataIndex1, dataKeyIndex, detectChanges, isLatest); + } + }); + } } private resetData() { + this.data = []; this.datasourceData = []; this.entityIdToDataIndex = {}; for (let dataIndex = 0; dataIndex < this.pageData.data.length; dataIndex++) { @@ -609,19 +755,18 @@ export class EntityDataSubscription { this.pageData = pageData; if (this.entityDataSubscriptionOptions.isPaginatedDataSubscription) { - this.prepareData(); + this.prepareData(false); } else if (isInitialData) { this.resetData(); } - const data: Array> = []; for (let dataIndex = 0; dataIndex < pageData.data.length; dataIndex++) { const entityData = pageData.data[dataIndex]; this.processEntityData(entityData, dataIndex, false, (data1, dataIndex1, dataKeyIndex) => { - if (!data[dataIndex1]) { - data[dataIndex1] = []; + if (!this.data[dataIndex1]) { + this.data[dataIndex1] = []; } - data[dataIndex1][dataKeyIndex] = data1; + this.data[dataIndex1][dataKeyIndex] = data1; } ); } @@ -630,7 +775,7 @@ export class EntityDataSubscription { this.entityDataResolveSubject.next( { pageData, - data, + data: this.data, datasourceIndex: this.listener.configDatasourceIndex, pageLink: this.entityDataSubscriptionOptions.pageLink } @@ -638,7 +783,7 @@ export class EntityDataSubscription { this.entityDataResolveSubject.complete(); } else { if (isInitialData || this.entityDataSubscriptionOptions.isPaginatedDataSubscription) { - this.listener.dataLoaded(pageData, data, + this.listener.dataLoaded(pageData, this.data, this.listener.configDatasourceIndex, this.entityDataSubscriptionOptions.pageLink); } if (this.entityDataSubscriptionOptions.isPaginatedDataSubscription && isInitialData) { @@ -648,7 +793,7 @@ export class EntityDataSubscription { this.entityDataResolveSubject.next( { pageData, - data, + data: this.data, datasourceIndex: this.listener.configDatasourceIndex, pageLink: this.entityDataSubscriptionOptions.pageLink } @@ -675,27 +820,85 @@ export class EntityDataSubscription { private processEntityData(entityData: EntityData, dataIndex: number, isUpdate: boolean, dataUpdatedCb: DataUpdatedCb) { - if ((this.entityDataSubscriptionOptions.type === widgetType.latest || - this.entityDataSubscriptionOptions.type === widgetType.timeseries) && entityData.latest) { - for (const type of Object.keys(entityData.latest)) { - const subscriptionData = this.toSubscriptionData(entityData.latest[type], false); - const dataKeyType = entityKeyTypeToDataKeyType(EntityKeyType[type]); - this.onData(subscriptionData, dataKeyType, dataIndex, true, - this.entityDataSubscriptionOptions.type === widgetType.timeseries, dataUpdatedCb); + if (this.entityDataSubscriptionOptions.type === widgetType.latest || + this.entityDataSubscriptionOptions.type === widgetType.timeseries) { + if (entityData.aggLatest) { + const aggData: IndexedSubscriptionData = []; + for (const idStr of Object.keys(entityData.aggLatest)) { + const id = Number(idStr); + const dataKey = this.dataKeyByIndex(id); + const aggLatestData = entityData.aggLatest[id]; + if (dataKey.comparisonEnabled) { + const keyData = EntityDataSubscription.calculateComparisonValue(dataKey, aggLatestData); + this.onKeyData(keyData, dataKey.name, id, dataKey.type, dataIndex, true, + this.entityDataSubscriptionOptions.type === widgetType.timeseries, true, dataUpdatedCb); + } else { + aggData[id] = [[aggLatestData.current.ts, aggLatestData.current.value, aggLatestData.current.count]]; + } + } + if (Object.keys(aggData).length > 0 && this.tsLatestDataAggregators && this.tsLatestDataAggregators[dataIndex]) { + const dataAggregator = this.tsLatestDataAggregators[dataIndex]; + let prevDataCb; + if (!isUpdate) { + prevDataCb = dataAggregator.updateOnDataCb((data, detectChanges) => { + this.onIndexedData(data, dataIndex, detectChanges, + this.entityDataSubscriptionOptions.type === widgetType.timeseries, dataUpdatedCb); + }); + } + dataAggregator.onData(aggData, false, this.history, true); + if (prevDataCb) { + dataAggregator.updateOnDataCb(prevDataCb); + } + } + } + if (entityData.latest) { + for (const type of Object.keys(entityData.latest)) { + const subscriptionData = this.toSubscriptionData(entityData.latest[type], false); + const dataKeyType = entityKeyTypeToDataKeyType(EntityKeyType[type]); + if (isUpdate && EntityKeyType[type] === EntityKeyType.TIME_SERIES) { + const keys: string[] = Object.keys(subscriptionData); + const latestTsKeys = this.latestValues.filter(key => key.type === EntityKeyType.TIME_SERIES && keys.includes(key.key)); + if (latestTsKeys.length) { + const latestTsSubsciptionData: SubscriptionData = {}; + for (const latestTsKey of latestTsKeys) { + latestTsSubsciptionData[latestTsKey.key] = subscriptionData[latestTsKey.key]; + } + this.onData(latestTsSubsciptionData, dataKeyType, dataIndex, true, + this.entityDataSubscriptionOptions.type === widgetType.timeseries, dataUpdatedCb); + } + const aggTsKeys = this.aggTsValues.filter(key => keys.includes(key.key)); + if (!this.history && aggTsKeys.length && this.tsLatestDataAggregators && this.tsLatestDataAggregators[dataIndex]) { + const dataAggregator = this.tsLatestDataAggregators[dataIndex]; + const indexedData: IndexedSubscriptionData = []; + for (const aggKey of aggTsKeys) { + indexedData[aggKey.id] = subscriptionData[aggKey.key]; + } + dataAggregator.onData(indexedData, true, false, true); + } + } else { + this.onData(subscriptionData, dataKeyType, dataIndex, true, + this.entityDataSubscriptionOptions.type === widgetType.timeseries, dataUpdatedCb); + } + } } } if (this.entityDataSubscriptionOptions.type === widgetType.timeseries && entityData.timeseries) { const subscriptionData = this.toSubscriptionData(entityData.timeseries, true); if (this.dataAggregators && this.dataAggregators[dataIndex]) { const dataAggregator = this.dataAggregators[dataIndex]; + const keyNames = Object.keys(subscriptionData); + const dataKeys = this.timeseriesDataKeysByKeyNames(keyNames); + const indexedData: IndexedSubscriptionData = []; + for (const dataKey of dataKeys) { + indexedData[dataKey.index] = subscriptionData[dataKey.name]; + } let prevDataCb; if (!isUpdate) { prevDataCb = dataAggregator.updateOnDataCb((data, detectChanges) => { - this.onData(data, this.datasourceType === DatasourceType.function ? - DataKeyType.function : DataKeyType.timeseries, dataIndex, detectChanges, false, dataUpdatedCb); + this.onIndexedData(data, dataIndex, detectChanges, false, dataUpdatedCb); }); } - dataAggregator.onData({data: subscriptionData}, false, this.history, true); + dataAggregator.onData(indexedData, false, this.history, true); if (prevDataCb) { dataAggregator.updateOnDataCb(prevDataCb); } @@ -707,92 +910,109 @@ export class EntityDataSubscription { private onData(sourceData: SubscriptionData, type: DataKeyType, dataIndex: number, detectChanges: boolean, isTsLatest: boolean, dataUpdatedCb: DataUpdatedCb) { - for (const keyName of Object.keys(sourceData)) { - const keyData = sourceData[keyName]; - const key = `${keyName}_${type}${isTsLatest ? '_latest' : ''}`; - const dataKeyList = this.dataKeys[key] as Array; - for (let keyIndex = 0; dataKeyList && keyIndex < dataKeyList.length; keyIndex++) { - const datasourceKey = `${key}_${keyIndex}`; - if (this.datasourceData[dataIndex][datasourceKey].data) { - const dataKey = dataKeyList[keyIndex]; - const data: DataSet = []; - let prevSeries: [number, any]; - let prevOrigSeries: [number, any]; - let datasourceKeyData: DataSet; - let datasourceOrigKeyData: DataSet; - let update = false; - if (this.realtime && !isTsLatest) { - datasourceKeyData = []; - datasourceOrigKeyData = []; - } else { - datasourceKeyData = this.datasourceData[dataIndex][datasourceKey].data; - datasourceOrigKeyData = this.datasourceOrigData[dataIndex][datasourceKey].data; - } - if (datasourceKeyData.length > 0) { - prevSeries = datasourceKeyData[datasourceKeyData.length - 1]; - prevOrigSeries = datasourceOrigKeyData[datasourceOrigKeyData.length - 1]; - } else { - prevSeries = [0, 0]; - prevOrigSeries = [0, 0]; - } - this.datasourceOrigData[dataIndex][datasourceKey].data = []; - if (this.entityDataSubscriptionOptions.type === widgetType.timeseries && !isTsLatest) { - keyData.forEach((keySeries) => { - let series = keySeries; - const time = series[0]; - this.datasourceOrigData[dataIndex][datasourceKey].data.push(series); - let value = this.convertValue(series[1]); - if (dataKey.postFunc) { - value = dataKey.postFunc(time, value, prevSeries[1], prevOrigSeries[0], prevOrigSeries[1]); - } - prevOrigSeries = series; - series = [time, value]; - data.push(series); - prevSeries = series; - }); - update = true; - } else if (this.entityDataSubscriptionOptions.type === widgetType.latest || isTsLatest) { - if (keyData.length > 0) { - let series = keyData[0]; - const time = series[0]; - this.datasourceOrigData[dataIndex][datasourceKey].data.push(series); - let value = this.convertValue(series[1]); - if (dataKey.postFunc) { - value = dataKey.postFunc(time, value, prevSeries[1], prevOrigSeries[0], prevOrigSeries[1]); - } - series = [time, value]; - data.push(series); - } - update = true; - } - if (update) { - this.datasourceData[dataIndex][datasourceKey].data = data; - dataUpdatedCb(this.datasourceData[dataIndex][datasourceKey], dataIndex, dataKey.index, detectChanges, isTsLatest); - } - } + for (const key of Object.keys(sourceData)) { + const keyData = sourceData[key]; + this.onKeyData(keyData, key, 0, type, + dataIndex, detectChanges, isTsLatest, false, dataUpdatedCb); + } + } + + private onIndexedData(sourceData: IndexedSubscriptionData, dataIndex: number, detectChanges: boolean, + isTsLatest: boolean, dataUpdatedCb: DataUpdatedCb) { + for (const indexStr of Object.keys(sourceData)) { + const id = Number(indexStr); + const dataKey = this.dataKeyByIndex(id); + const isAggLatest = dataKey.aggregationType && dataKey.aggregationType !== AggregationType.NONE; + const keyData = sourceData[id]; + let keyName = dataKey.name; + if (dataKey.type === DataKeyType.function) { + keyName += `_${dataKey.index}`; } + this.onKeyData(keyData, keyName, id, dataKey.type, + dataIndex, detectChanges, isTsLatest, isAggLatest, dataUpdatedCb); } } - private convertValue(val: string): any { - if (val && isNumeric(val) && Number(val).toString() === val) { - return Number(val); + private onKeyData(keyData: [number, any, number?][], keyName: string, id: number, type: DataKeyType, + dataIndex: number, detectChanges: boolean, + isTsLatest: boolean, isAggLatest: boolean, dataUpdatedCb: DataUpdatedCb) { + const keyIdSuffix = isAggLatest ? `_${id}` : ''; + const key = `${keyName}_${type}${keyIdSuffix}${isTsLatest ? '_latest' : ''}`; + const dataKeyList = this.dataKeys[key] as Array; + for (let keyIndex = 0; dataKeyList && keyIndex < dataKeyList.length; keyIndex++) { + const datasourceKey = `${key}_${keyIndex}`; + if (this.datasourceData[dataIndex][datasourceKey].data) { + const dataKey = dataKeyList[keyIndex]; + const data: DataSet = []; + let prevSeries: [number, any]; + let prevOrigSeries: [number, any]; + let datasourceKeyData: DataSet; + let datasourceOrigKeyData: DataSet; + let update = false; + if (this.realtime && !isTsLatest) { + datasourceKeyData = []; + datasourceOrigKeyData = []; + } else { + datasourceKeyData = this.datasourceData[dataIndex][datasourceKey].data; + datasourceOrigKeyData = this.datasourceOrigData[dataIndex][datasourceKey].data; + } + if (datasourceKeyData.length > 0) { + prevSeries = datasourceKeyData[datasourceKeyData.length - 1]; + prevOrigSeries = datasourceOrigKeyData[datasourceOrigKeyData.length - 1]; + } else { + prevSeries = [0, 0]; + prevOrigSeries = [0, 0]; + } + this.datasourceOrigData[dataIndex][datasourceKey].data = []; + if (this.entityDataSubscriptionOptions.type === widgetType.timeseries && !isTsLatest) { + keyData.forEach((keySeries) => { + let series = keySeries; + const time = series[0]; + this.datasourceOrigData[dataIndex][datasourceKey].data.push([series[0], series[1]]); + let value = EntityDataSubscription.convertValue(series[1]); + if (dataKey.postFunc) { + value = dataKey.postFunc(time, value, prevSeries[1], prevOrigSeries[0], prevOrigSeries[1]); + } + prevOrigSeries = [series[0], series[1]]; + series = [series[0], value]; + data.push([series[0], series[1]]); + prevSeries = [series[0], series[1]]; + }); + update = true; + } else if (this.entityDataSubscriptionOptions.type === widgetType.latest || isTsLatest) { + if (keyData.length > 0) { + let series = keyData[0]; + const time = series[0]; + this.datasourceOrigData[dataIndex][datasourceKey].data.push([series[0], series[1]]); + let value = EntityDataSubscription.convertValue(series[1]); + if (dataKey.postFunc) { + value = dataKey.postFunc(time, value, prevSeries[1], prevOrigSeries[0], prevOrigSeries[1]); + } + series = [time, value]; + data.push([series[0], series[1]]); + } + update = true; + } + if (update) { + this.datasourceData[dataIndex][datasourceKey].data = data; + dataUpdatedCb(this.datasourceData[dataIndex][datasourceKey], dataIndex, dataKey.index, detectChanges, isTsLatest); + } + } } - return val; } private toSubscriptionData(sourceData: {[key: string]: TsValue | TsValue[]}, isTs: boolean): SubscriptionData { const subsData: SubscriptionData = {}; for (const keyName of Object.keys(sourceData)) { const values = sourceData[keyName]; - const dataSet: [number, any][] = []; + const dataSet: [number, any, number?][] = []; if (isTs) { (values as TsValue[]).forEach((keySeries) => { - dataSet.push([keySeries.ts, keySeries.value]); + dataSet.push([keySeries.ts, keySeries.value, keySeries.count]); }); } else { const tsValue = values as TsValue; - dataSet.push([tsValue.ts, tsValue.value]); + dataSet.push([tsValue.ts, tsValue.value, tsValue.count]); } subsData[keyName] = dataSet; } @@ -800,21 +1020,37 @@ export class EntityDataSubscription { } private createRealtimeDataAggregator(subsTw: SubscriptionTimewindow, - tsKeyNames: Array, - dataKeyType: DataKeyType, + tsKeys: Array, + isLatestDataAgg: boolean, dataIndex: number, dataUpdatedCb: DataUpdatedCb): DataAggregator { return new DataAggregator( (data, detectChanges) => { - this.onData(data, dataKeyType, dataIndex, detectChanges, false, dataUpdatedCb); + this.onIndexedData(data, dataIndex, detectChanges, + isLatestDataAgg && (this.entityDataSubscriptionOptions.type === widgetType.timeseries), dataUpdatedCb); }, - tsKeyNames, + tsKeys, + isLatestDataAgg, subsTw, this.utils, - this.entityDataSubscriptionOptions.ignoreDataUpdateOnIntervalTick + this.entityDataSubscriptionOptions.ignoreDataUpdateOnIntervalTick || isLatestDataAgg ); } + private dataKeyByIndex(index: number): SubscriptionDataKey { + return this.dataKeysList.find(key => key.index === index); + } + + private timeseriesDataKeysByKeyNames(keyNames: string[]): SubscriptionDataKey[] { + const result: SubscriptionDataKey[] = []; + for (const keyName of keyNames) { + const key = `${keyName}_${DataKeyType.timeseries}`; + const dataKeyList = this.dataKeys[key] as Array; + result.push(...dataKeyList); + } + return result; + } + private generateSeries(dataKey: SubscriptionDataKey, startTime: number, endTime: number): [number, any][] { const data: [number, any][] = []; let prevSeries: [number, any]; @@ -893,9 +1129,7 @@ export class EntityDataSubscription { let startTime: number; let endTime: number; let delta: number; - const generatedData: SubscriptionDataHolder = { - data: {} - }; + const generatedData: IndexedSubscriptionData = []; if (!this.history) { delta = Math.floor(this.tickElapsed / this.frequency); } @@ -928,7 +1162,7 @@ export class EntityDataSubscription { endTime = Math.min(currentTime, endTime); } } - generatedData.data[`${dataKey.name}_${dataKey.index}`] = this.generateSeries(dataKey, startTime, endTime); + generatedData[dataKey.index] = this.generateSeries(dataKey, startTime, endTime); } if (this.dataAggregators && this.dataAggregators.length) { this.dataAggregators[0].onData(generatedData, true, this.history, detectChanges); diff --git a/ui-ngx/src/app/core/api/entity-data.service.ts b/ui-ngx/src/app/core/api/entity-data.service.ts index 3d3112f007..6d8ea94dc0 100644 --- a/ui-ngx/src/app/core/api/entity-data.service.ts +++ b/ui-ngx/src/app/core/api/entity-data.service.ts @@ -31,6 +31,7 @@ import { Observable, of } from 'rxjs'; export interface EntityDataListener { subscriptionType: widgetType; + useTimewindow?: boolean; subscriptionTimewindow?: SubscriptionTimewindow; latestTsOffset?: number; configDatasource: Datasource; @@ -73,6 +74,21 @@ export class EntityDataService { } } + private static toSubscriptionDataKey(dataKey: DataKey, latest: boolean): SubscriptionDataKey { + return { + name: dataKey.name, + type: dataKey.type, + aggregationType: dataKey.aggregationType, + comparisonEnabled: dataKey.comparisonEnabled, + timeForComparison: dataKey.timeForComparison, + comparisonCustomIntervalValue: dataKey.comparisonCustomIntervalValue, + comparisonResultType: dataKey.comparisonResultType, + funcBody: dataKey.funcBody, + postFuncBody: dataKey.postFuncBody, + latest + }; + } + public prepareSubscription(listener: EntityDataListener, ignoreDataUpdateOnIntervalTick = false): Observable { const datasource = listener.configDatasource; @@ -93,10 +109,10 @@ export class EntityDataService { public startSubscription(listener: EntityDataListener) { if (listener.subscription) { - if (listener.subscriptionType === widgetType.timeseries) { + if (listener.useTimewindow) { listener.subscriptionOptions.subscriptionTimewindow = deepClone(listener.subscriptionTimewindow); - listener.subscriptionOptions.latestTsOffset = listener.latestTsOffset; - } else if (listener.subscriptionType === widgetType.latest) { + } + if (listener.subscriptionType === widgetType.timeseries || listener.subscriptionType === widgetType.latest) { listener.subscriptionOptions.latestTsOffset = listener.latestTsOffset; } listener.subscription.start(); @@ -122,10 +138,10 @@ export class EntityDataService { return of(null); } listener.subscription = new EntityDataSubscription(listener, this.telemetryService, this.utils); - if (listener.subscriptionType === widgetType.timeseries) { + if (listener.useTimewindow) { listener.subscriptionOptions.subscriptionTimewindow = deepClone(listener.subscriptionTimewindow); - listener.subscriptionOptions.latestTsOffset = listener.latestTsOffset; - } else if (listener.subscriptionType === widgetType.latest) { + } + if (listener.subscriptionType === widgetType.timeseries || listener.subscriptionType === widgetType.latest) { listener.subscriptionOptions.latestTsOffset = listener.latestTsOffset; } return listener.subscription.subscribe(); @@ -146,11 +162,11 @@ export class EntityDataService { ignoreDataUpdateOnIntervalTick: boolean): EntityDataSubscriptionOptions { const subscriptionDataKeys: Array = []; datasource.dataKeys.forEach((dataKey) => { - subscriptionDataKeys.push(this.toSubscriptionDataKey(dataKey, false)); + subscriptionDataKeys.push(EntityDataService.toSubscriptionDataKey(dataKey, false)); }); if (datasource.latestDataKeys) { datasource.latestDataKeys.forEach((dataKey) => { - subscriptionDataKeys.push(this.toSubscriptionDataKey(dataKey, true)); + subscriptionDataKeys.push(EntityDataService.toSubscriptionDataKey(dataKey, true)); }); } const entityDataSubscriptionOptions: EntityDataSubscriptionOptions = { @@ -171,14 +187,4 @@ export class EntityDataService { entityDataSubscriptionOptions.ignoreDataUpdateOnIntervalTick = ignoreDataUpdateOnIntervalTick; return entityDataSubscriptionOptions; } - - private toSubscriptionDataKey(dataKey: DataKey, latest: boolean): SubscriptionDataKey { - return { - name: dataKey.name, - type: dataKey.type, - funcBody: dataKey.funcBody, - postFuncBody: dataKey.postFuncBody, - latest - }; - } } diff --git a/ui-ngx/src/app/core/api/widget-subscription.ts b/ui-ngx/src/app/core/api/widget-subscription.ts index 41aeb5dd14..a01e731b8d 100644 --- a/ui-ngx/src/app/core/api/widget-subscription.ts +++ b/ui-ngx/src/app/core/api/widget-subscription.ts @@ -28,6 +28,7 @@ import { DataSetHolder, Datasource, DatasourceData, + datasourcesHasAggregation, DatasourceType, LegendConfig, LegendData, @@ -95,6 +96,7 @@ export class WidgetSubscription implements IWidgetSubscription { timezone: string; subscriptionTimewindow: SubscriptionTimewindow; useDashboardTimewindow: boolean; + useTimewindow: boolean; tsOffset = 0; hasDataPageLink: boolean; @@ -200,6 +202,7 @@ export class WidgetSubscription implements IWidgetSubscription { this.originalTimewindow = null; this.timeWindow = {}; this.useDashboardTimewindow = options.useDashboardTimewindow; + this.useTimewindow = true; if (this.useDashboardTimewindow) { this.timeWindowConfig = deepClone(options.dashboardTimewindow); } else { @@ -245,15 +248,16 @@ export class WidgetSubscription implements IWidgetSubscription { this.timeWindow = {}; this.useDashboardTimewindow = options.useDashboardTimewindow; this.stateData = options.stateData; - if (this.type === widgetType.latest) { - this.timezone = options.dashboardTimewindow.timezone; - this.updateTsOffset(); - } + this.useTimewindow = this.type === widgetType.timeseries || datasourcesHasAggregation(this.configuredDatasources); if (this.useDashboardTimewindow) { this.timeWindowConfig = deepClone(options.dashboardTimewindow); } else { this.timeWindowConfig = deepClone(options.timeWindowConfig); } + if (this.type === widgetType.latest) { + this.timezone = this.useTimewindow ? this.timeWindowConfig.timezone : options.dashboardTimewindow.timezone; + this.updateTsOffset(); + } this.subscriptionTimewindow = null; this.comparisonEnabled = options.comparisonEnabled && isHistoryTypeTimewindow(this.timeWindowConfig); @@ -443,6 +447,7 @@ export class WidgetSubscription implements IWidgetSubscription { const resolveResultObservables = this.configuredDatasources.map((datasource, index) => { const listener: EntityDataListener = { subscriptionType: this.type, + useTimewindow: this.useTimewindow, configDatasource: datasource, configDatasourceIndex: index, dataLoaded: (pageData, data1, datasourceIndex, pageLink) => { @@ -626,22 +631,31 @@ export class WidgetSubscription implements IWidgetSubscription { } onDashboardTimewindowChanged(newDashboardTimewindow: Timewindow) { - if (this.type === widgetType.timeseries || this.type === widgetType.alarm) { + let doUpdate = false; + let isTimewindowTypeChanged = false; + if (this.useTimewindow) { if (this.useDashboardTimewindow) { + if (this.type === widgetType.latest) { + if (newDashboardTimewindow && this.timezone !== newDashboardTimewindow.timezone) { + this.timezone = newDashboardTimewindow.timezone; + doUpdate = this.updateTsOffset(); + } + } if (!isEqual(this.timeWindowConfig, newDashboardTimewindow) && newDashboardTimewindow) { - const isTimewindowTypeChanged = timewindowTypeChanged(this.timeWindowConfig, newDashboardTimewindow); + isTimewindowTypeChanged = timewindowTypeChanged(this.timeWindowConfig, newDashboardTimewindow); this.timeWindowConfig = deepClone(newDashboardTimewindow); - this.update(isTimewindowTypeChanged); + doUpdate = true; } } } else if (this.type === widgetType.latest) { if (newDashboardTimewindow && this.timezone !== newDashboardTimewindow.timezone) { this.timezone = newDashboardTimewindow.timezone; - if (this.updateTsOffset()) { - this.update(); - } + doUpdate = this.updateTsOffset(); } } + if (doUpdate) { + this.update(isTimewindowTypeChanged); + } } updateDataVisibility(index: number): void { @@ -660,6 +674,12 @@ export class WidgetSubscription implements IWidgetSubscription { updateTimewindowConfig(newTimewindow: Timewindow): void { if (!this.useDashboardTimewindow) { + if (this.type === widgetType.latest) { + if (newTimewindow && this.timezone !== newTimewindow.timezone) { + this.timezone = newTimewindow.timezone; + this.updateTsOffset(); + } + } const isTimewindowTypeChanged = timewindowTypeChanged(this.timeWindowConfig, newTimewindow); this.timeWindowConfig = newTimewindow; this.update(isTimewindowTypeChanged); @@ -874,11 +894,12 @@ export class WidgetSubscription implements IWidgetSubscription { } const datasource = this.configuredDatasources[datasourceIndex]; if (datasource) { - if (this.type === widgetType.timeseries && this.timeWindowConfig) { + if (this.useTimewindow && this.timeWindowConfig) { this.updateRealtimeSubscription(); } entityDataListener = { subscriptionType: this.type, + useTimewindow: this.useTimewindow, configDatasource: datasource, configDatasourceIndex: datasourceIndex, subscriptionTimewindow: this.subscriptionTimewindow, @@ -940,7 +961,7 @@ export class WidgetSubscription implements IWidgetSubscription { private updateDataTimewindow() { if (!this.hasDataPageLink) { - if (this.type === widgetType.timeseries && this.timeWindowConfig) { + if (this.useTimewindow && this.timeWindowConfig) { this.updateRealtimeSubscription(); if (this.comparisonEnabled) { this.updateSubscriptionForComparison(); @@ -952,11 +973,11 @@ export class WidgetSubscription implements IWidgetSubscription { private dataSubscribe() { this.updateDataTimewindow(); if (!this.hasDataPageLink) { - if (this.type === widgetType.timeseries && this.timeWindowConfig && this.subscriptionTimewindow.fixedWindow) { + if (this.useTimewindow && this.timeWindowConfig && this.subscriptionTimewindow.fixedWindow) { this.onDataUpdated(); } const forceUpdate = !this.datasources.length; - const notifyDataLoaded = !this.entityDataListeners.filter((listener) => listener.subscription ? true : false).length; + const notifyDataLoaded = !this.entityDataListeners.filter((listener) => !!listener.subscription).length; this.entityDataListeners.forEach((listener) => { if (this.comparisonEnabled && listener.configDatasource.isAdditional) { listener.subscriptionTimewindow = this.timewindowForComparison; diff --git a/ui-ngx/src/app/core/services/dashboard-utils.service.ts b/ui-ngx/src/app/core/services/dashboard-utils.service.ts index c157ca869e..90238ebaba 100644 --- a/ui-ngx/src/app/core/services/dashboard-utils.service.ts +++ b/ui-ngx/src/app/core/services/dashboard-utils.service.ts @@ -34,6 +34,7 @@ import { Datasource, DatasourceType, Widget, widgetType } from '@app/shared/mode import { EntityType } from '@shared/models/entity-type.models'; import { AliasFilterType, EntityAlias, EntityAliasFilter } from '@app/shared/models/alias.models'; import { EntityId } from '@app/shared/models/id/entity-id'; +import { initModelFromDefaultTimewindow } from '@shared/models/time/time.models'; @Injectable({ providedIn: 'root' @@ -215,6 +216,9 @@ export class DashboardUtilsService { delete datasource.deviceAliasId; } }); + if (widget.type === widgetType.latest) { + widget.config.timewindow = initModelFromDefaultTimewindow(widget.config.timewindow, true, this.timeService); + } // Temp workaround if (widget.isSystemType && widget.bundleAlias === 'charts' && widget.typeAlias === 'timeseries') { widget.typeAlias = 'basic_timeseries'; diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts index 4ebaccb774..c145794163 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts @@ -1062,7 +1062,7 @@ export class DashboardPageComponent extends PageComponent implements IDashboardC dataKeys: config.alarmSource.dataKeys || [] }; } - const newWidget: Widget = { + let newWidget: Widget = { isSystemType: widget.isSystemType, bundleAlias: widget.bundleAlias, typeAlias: widgetTypeInfo.alias, @@ -1076,6 +1076,7 @@ export class DashboardPageComponent extends PageComponent implements IDashboardC row: 0, col: 0 }; + newWidget = this.dashboardUtils.validateAndUpdateWidget(newWidget); if (widgetTypeInfo.typeParameters.useCustomDatasources) { this.addWidgetToDashboard(newWidget); } else { diff --git a/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.html b/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.html index 4f85de08b0..2327d708da 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.html @@ -36,6 +36,7 @@ [dashboard]="data.dashboard" [aliasController]="data.aliasController" [widget]="data.widget" + [widgetType]="data.widgetType" [showPostProcessing]="data.showPostProcessing" [callbacks]="data.callbacks" formControlName="dataKey"> diff --git a/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.ts b/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.ts index 5c67242958..a3d5d8fd65 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/data-key-config-dialog.component.ts @@ -22,7 +22,7 @@ import { AppState } from '@core/core.state'; import { FormBuilder, FormControl, FormGroup, FormGroupDirective, NgForm, Validators } from '@angular/forms'; import { Router } from '@angular/router'; import { DialogComponent } from '@shared/components/dialog.component'; -import { DataKey, Widget } from '@shared/models/widget.models'; +import { DataKey, Widget, widgetType } from '@shared/models/widget.models'; import { DataKeysCallbacks } from './data-keys.component.models'; import { DataKeyConfigComponent } from '@home/components/widget/data-key-config.component'; import { Dashboard } from '@shared/models/dashboard.models'; @@ -35,6 +35,7 @@ export interface DataKeyConfigDialogData { dashboard: Dashboard; aliasController: IAliasController; widget: Widget; + widgetType: widgetType; entityAliasId?: string; showPostProcessing?: boolean; callbacks?: DataKeysCallbacks; diff --git a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html index 19cc902098..9088f5cc33 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html @@ -62,6 +62,75 @@ +
+ + datakey.aggregation + + + {{ aggregationTypesTranslations.get(aggregationTypes[aggregation]) | translate }} + + + + {{ dataKeyFormGroup.get('aggregationType').value ? (dataKeyAggregationTypeHintTranslations.get(aggregationTypes[dataKeyFormGroup.get('aggregationType').value]) | translate) : '' }} +
+ {{ 'datakey.aggregation-type-hint-common' | translate }} +
+ + +
+ datakey.delta-calculation + + + + + {{ 'datakey.enable-delta-calculation' | translate }} + + {{ 'datakey.enable-delta-calculation-hint' | translate }} + + + +
+ + widgets.chart.time-for-comparison + + + {{ 'widgets.chart.time-for-comparison-previous-interval' | translate }} + + + {{ 'widgets.chart.time-for-comparison-days' | translate }} + + + {{ 'widgets.chart.time-for-comparison-weeks' | translate }} + + + {{ 'widgets.chart.time-for-comparison-months' | translate }} + + + {{ 'widgets.chart.time-for-comparison-years' | translate }} + + + {{ 'widgets.chart.time-for-comparison-custom-interval' | translate }} + + + + + widgets.chart.custom-interval-value + + + + datakey.delta-calculation-result + + + {{ comparisonResultTypeTranslations.get(comparisonResultTypes[comparisonResultType]) | translate }} + + + +
+ + +
+
datakey.data-generation-func
diff --git a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.scss b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.scss index 8b71ee3664..7fd1fba872 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.scss +++ b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.scss @@ -28,6 +28,36 @@ padding-left: 12px; } } + + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + + &.fields-group-slider { + padding: 0; + + legend { + margin-left: 16px; + } + + > .tb-settings { + margin-top: 0; + padding: 0 16px 8px; + } + } + } } } @@ -42,5 +72,50 @@ } } } + .mat-expansion-panel { + &.tb-settings { + box-shadow: none; + + .mat-content { + overflow: visible; + } + + .mat-expansion-panel-header { + padding: 0; + color: rgba(0, 0, 0, 0.87); + height: 140px; + + &:hover { + background: none; + } + + .mat-expansion-indicator { + padding: 2px; + } + } + + .mat-expansion-panel-header-description { + align-items: center; + } + + > .mat-expansion-panel-content { + > .mat-expansion-panel-body { + padding: 0; + } + } + } + + .mat-expansion-panel-content { + font: inherit; + } + } + + .mat-slide { + margin: 8px 0; + } + + .mat-slide-toggle-content { + white-space: normal; + } } } diff --git a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.ts index b97f17f0e0..a461796111 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.ts @@ -18,7 +18,13 @@ import { Component, ElementRef, forwardRef, Input, OnInit, ViewChild } from '@an import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { DataKey, Widget } from '@shared/models/widget.models'; +import { + ComparisonResultType, comparisonResultTypeTranslationMap, + DataKey, + dataKeyAggregationTypeHintTranslationMap, + Widget, + widgetType +} from '@shared/models/widget.models'; import { ControlValueAccessor, FormBuilder, @@ -43,6 +49,7 @@ import { JsonFormComponentData } from '@shared/components/json-form/json-form-co import { WidgetService } from '@core/http/widget.service'; import { Dashboard } from '@shared/models/dashboard.models'; import { IAliasController } from '@core/api/widget-api.models'; +import { aggregationTranslations, AggregationType, ComparisonDuration } from '@shared/models/time/time.models'; @Component({ selector: 'tb-data-key-config', @@ -65,6 +72,22 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con dataKeyTypes = DataKeyType; + widgetTypes = widgetType; + + aggregations = [AggregationType.NONE, ...Object.keys(AggregationType).filter(type => type !== AggregationType.NONE)]; + + aggregationTypes = AggregationType; + + aggregationTypesTranslations = aggregationTranslations; + + dataKeyAggregationTypeHintTranslations = dataKeyAggregationTypeHintTranslationMap; + + comparisonResultTypes = ComparisonResultType; + + comparisonResults = Object.keys(ComparisonResultType); + + comparisonResultTypeTranslations = comparisonResultTypeTranslationMap; + @Input() entityAliasId: string; @@ -80,6 +103,9 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con @Input() widget: Widget; + @Input() + widgetType: widgetType; + @Input() dataKeySettingsSchema: any; @@ -155,6 +181,11 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con } this.dataKeyFormGroup = this.fb.group({ name: [null, []], + aggregationType: [null, []], + comparisonEnabled: [null, []], + timeForComparison: [null, [Validators.required]], + comparisonCustomIntervalValue: [null, [Validators.required, Validators.min(1000)]], + comparisonResultType: [null, [Validators.required]], label: [null, [Validators.required]], color: [null, [Validators.required]], units: [null, []], @@ -164,6 +195,32 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con postFuncBody: [null, []] }); + this.dataKeyFormGroup.get('aggregationType').valueChanges.subscribe( + (aggType) => { + if (!this.dataKeyFormGroup.get('label').dirty) { + let newLabel = this.dataKeyFormGroup.get('name').value; + if (aggType !== AggregationType.NONE) { + const prefix = this.translate.instant(aggregationTranslations.get(aggType)); + newLabel = prefix + ' ' + newLabel; + } + this.dataKeyFormGroup.get('label').patchValue(newLabel); + } + this.updateComparisonValidators(); + } + ); + + this.dataKeyFormGroup.get('comparisonEnabled').valueChanges.subscribe( + () => { + this.updateComparisonValues(); + } + ); + + this.dataKeyFormGroup.get('timeForComparison').valueChanges.subscribe( + () => { + this.updateComparisonValues(); + } + ); + this.dataKeyFormGroup.valueChanges.subscribe(() => { this.updateModel(); }); @@ -199,22 +256,86 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con if (this.modelValue.postFuncBody && this.modelValue.postFuncBody.length) { this.modelValue.usePostProcessing = true; } + if (this.widgetType === widgetType.latest && this.modelValue.type === DataKeyType.timeseries && !this.modelValue.aggregationType) { + this.modelValue.aggregationType = AggregationType.NONE; + } this.dataKeyFormGroup.patchValue(this.modelValue, {emitEvent: false}); + this.updateValidators(); + if (this.displayAdvanced) { + this.dataKeySettingsData.model = this.modelValue.settings; + this.dataKeySettingsFormGroup.patchValue({ + settings: this.dataKeySettingsData + }, {emitEvent: false}); + } + } + + private updateValidators() { this.dataKeyFormGroup.get('name').setValidators(this.modelValue.type !== DataKeyType.function && - this.modelValue.type !== DataKeyType.count - ? [Validators.required] : []); + this.modelValue.type !== DataKeyType.count + ? [Validators.required] : []); if (this.modelValue.type === DataKeyType.count) { this.dataKeyFormGroup.get('name').disable({emitEvent: false}); } else { this.dataKeyFormGroup.get('name').enable({emitEvent: false}); } this.dataKeyFormGroup.get('name').updateValueAndValidity({emitEvent: false}); - if (this.displayAdvanced) { - this.dataKeySettingsData.model = this.modelValue.settings; - this.dataKeySettingsFormGroup.patchValue({ - settings: this.dataKeySettingsData - }, {emitEvent: false}); + this.updateComparisonValidators(); + } + + private updateComparisonValues() { + const comparisonEnabled = this.dataKeyFormGroup.get('comparisonEnabled').value; + if (comparisonEnabled) { + const timeForComparison: ComparisonDuration = this.dataKeyFormGroup.get('timeForComparison').value; + if (!timeForComparison) { + this.dataKeyFormGroup.get('timeForComparison').patchValue('previousInterval', {emitEvent: false}); + } else if (timeForComparison === 'customInterval') { + const comparisonCustomIntervalValue = this.dataKeyFormGroup.get('comparisonCustomIntervalValue').value; + if (!comparisonCustomIntervalValue) { + this.dataKeyFormGroup.get('comparisonCustomIntervalValue').patchValue(7200000, {emitEvent: false}); + } + } + const comparisonResultType: ComparisonResultType = this.dataKeyFormGroup.get('comparisonResultType').value; + if (!comparisonResultType) { + this.dataKeyFormGroup.get('comparisonResultType').patchValue(ComparisonResultType.DELTA_ABSOLUTE, {emitEvent: false}); + } + } + this.updateComparisonValidators(); + } + + private updateComparisonValidators() { + const aggregationType: AggregationType = this.dataKeyFormGroup.get('aggregationType').value; + if (aggregationType && aggregationType !== AggregationType.NONE) { + this.dataKeyFormGroup.get('comparisonEnabled').enable({emitEvent: false}); + const comparisonEnabled = this.dataKeyFormGroup.get('comparisonEnabled').value; + if (comparisonEnabled) { + this.dataKeyFormGroup.get('timeForComparison').enable({emitEvent: false}); + const timeForComparison: ComparisonDuration = this.dataKeyFormGroup.get('timeForComparison').value; + if (timeForComparison) { + this.dataKeyFormGroup.get('comparisonResultType').enable({emitEvent: false}); + if (timeForComparison === 'customInterval') { + this.dataKeyFormGroup.get('comparisonCustomIntervalValue').enable({emitEvent: false}); + } else { + this.dataKeyFormGroup.get('comparisonCustomIntervalValue').disable({emitEvent: false}); + } + } else { + this.dataKeyFormGroup.get('comparisonResultType').disable({emitEvent: false}); + this.dataKeyFormGroup.get('comparisonCustomIntervalValue').disable({emitEvent: false}); + } + } else { + this.dataKeyFormGroup.get('timeForComparison').disable({emitEvent: false}); + this.dataKeyFormGroup.get('comparisonResultType').disable({emitEvent: false}); + this.dataKeyFormGroup.get('comparisonCustomIntervalValue').disable({emitEvent: false}); + } + } else { + this.dataKeyFormGroup.get('comparisonEnabled').disable({emitEvent: false}); + this.dataKeyFormGroup.get('timeForComparison').disable({emitEvent: false}); + this.dataKeyFormGroup.get('comparisonResultType').disable({emitEvent: false}); + this.dataKeyFormGroup.get('comparisonCustomIntervalValue').disable({emitEvent: false}); } + this.dataKeyFormGroup.get('comparisonEnabled').updateValueAndValidity({emitEvent: false}); + this.dataKeyFormGroup.get('timeForComparison').updateValueAndValidity({emitEvent: false}); + this.dataKeyFormGroup.get('comparisonResultType').updateValueAndValidity({emitEvent: false}); + this.dataKeyFormGroup.get('comparisonCustomIntervalValue').updateValueAndValidity({emitEvent: false}); } private updateModel() { diff --git a/ui-ngx/src/app/modules/home/components/widget/data-keys.component.html b/ui-ngx/src/app/modules/home/components/widget/data-keys.component.html index 3527a5ca2e..5df377cf07 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-keys.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/data-keys.component.html @@ -58,12 +58,7 @@ {{key.label}}
:
-
- f({{key.name}}) - - {{key.name}} - -
+
diff --git a/ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts index db593e64f2..2286510ba3 100644 --- a/ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts @@ -20,7 +20,7 @@ import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { DataKey, - Datasource, + Datasource, datasourcesHasAggregation, datasourcesHasOnlyComparisonAggregation, DatasourceType, datasourceTypeTranslationMap, defaultLegendConfig, @@ -42,7 +42,7 @@ import { Validators } from '@angular/forms'; import { WidgetConfigComponentData } from '@home/models/widget-component.models'; -import { deepClone, isDefined, isObject, isUndefined } from '@app/core/utils'; +import { deepClone, isDefined, isObject } from '@app/core/utils'; import { alarmFields, AlarmSearchStatus, @@ -51,7 +51,7 @@ import { alarmSeverityTranslations } from '@shared/models/alarm.models'; import { IAliasController } from '@core/api/widget-api.models'; -import { EntityAlias, EntityAliases } from '@shared/models/alias.models'; +import { EntityAlias } from '@shared/models/alias.models'; import { UtilsService } from '@core/services/utils.service'; import { DataKeyType } from '@shared/models/telemetry/telemetry.models'; import { TranslateService } from '@ngx-translate/core'; @@ -67,13 +67,14 @@ import { MatDialog } from '@angular/material/dialog'; import { EntityService } from '@core/http/entity.service'; import { JsonFormComponentData } from '@shared/components/json-form/json-form-component.models'; import { WidgetActionsData } from './action/manage-widget-actions.component.models'; -import { Dashboard, DashboardState } from '@shared/models/dashboard.models'; +import { Dashboard } from '@shared/models/dashboard.models'; import { entityFields } from '@shared/models/entity.models'; -import { Filter, Filters } from '@shared/models/query/query.models'; +import { Filter } from '@shared/models/query/query.models'; import { FilterDialogComponent, FilterDialogData } from '@home/components/filter/filter-dialog.component'; import { COMMA, ENTER, SEMICOLON } from '@angular/cdk/keycodes'; import { MatChipInputEvent } from '@angular/material/chips'; import { CdkDragDrop } from '@angular/cdk/drag-drop'; +import { AggregationType } from '@shared/models/time/time.models'; const emptySettingsSchema: JsonSchema = { type: 'object', @@ -334,10 +335,10 @@ export class WidgetConfigComponent extends PageComponent implements OnInit, Cont this.targetDeviceSettings = this.fb.group({}); this.alarmSourceSettings = this.fb.group({}); this.advancedSettings = this.fb.group({}); - if (this.widgetType === widgetType.timeseries || this.widgetType === widgetType.alarm) { - this.dataSettings.addControl('useDashboardTimewindow', this.fb.control(null)); - this.dataSettings.addControl('displayTimewindow', this.fb.control(null)); - this.dataSettings.addControl('timewindow', this.fb.control(null)); + if (this.widgetType === widgetType.timeseries || this.widgetType === widgetType.alarm || this.widgetType === widgetType.latest) { + this.dataSettings.addControl('useDashboardTimewindow', this.fb.control(true)); + this.dataSettings.addControl('displayTimewindow', this.fb.control({value: true, disabled: true})); + this.dataSettings.addControl('timewindow', this.fb.control({value: null, disabled: true})); this.dataSettings.get('useDashboardTimewindow').valueChanges.subscribe((value: boolean) => { if (value) { this.dataSettings.get('displayTimewindow').disable({emitEvent: false}); @@ -467,7 +468,7 @@ export class WidgetConfigComponent extends PageComponent implements OnInit, Cont }, {emitEvent: false} ); - if (this.widgetType === widgetType.timeseries || this.widgetType === widgetType.alarm) { + if (this.widgetType === widgetType.timeseries || this.widgetType === widgetType.alarm || this.widgetType === widgetType.latest) { const useDashboardTimewindow = isDefined(config.useDashboardTimewindow) ? config.useDashboardTimewindow : true; this.dataSettings.patchValue( @@ -733,6 +734,24 @@ export class WidgetConfigComponent extends PageComponent implements OnInit, Cont !!this.modelValue.settingsDirective && !!this.modelValue.settingsDirective.length); } + public displayTimewindowConfig(): boolean { + if (this.widgetType === widgetType.timeseries || this.widgetType === widgetType.alarm) { + return true; + } else if (this.widgetType === widgetType.latest) { + const datasources = this.dataSettings.get('datasources').value; + return datasourcesHasAggregation(datasources); + } + } + + public onlyHistoryTimewindow(): boolean { + if (this.widgetType === widgetType.latest) { + const datasources = this.dataSettings.get('datasources').value; + return datasourcesHasOnlyComparisonAggregation(datasources); + } else { + return false; + } + } + public onDatasourceDrop(event: CdkDragDrop) { const datasourcesFormArray = this.datasourcesFormArray(); const datasourceForm = datasourcesFormArray.at(event.previousIndex); diff --git a/ui-ngx/src/app/modules/home/components/widget/widget-container.component.html b/ui-ngx/src/app/modules/home/components/widget/widget-container.component.html index a6520e02bd..ad8f65a48b 100644 --- a/ui-ngx/src/app/modules/home/components/widget/widget-container.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/widget-container.component.html @@ -46,6 +46,9 @@ ; @@ -325,6 +333,10 @@ export class DashboardWidget implements GridsterItem, IDashboardWidget { hasAggregation: boolean; + onlyQuickInterval: boolean; + + onlyHistoryTimewindow: boolean; + style: {[klass: string]: any}; showWidgetTitlePanel: boolean; @@ -420,12 +432,28 @@ export class DashboardWidget implements GridsterItem, IDashboardWidget { this.dropShadow = isDefined(this.widget.config.dropShadow) ? this.widget.config.dropShadow : true; this.enableFullscreen = isDefined(this.widget.config.enableFullscreen) ? this.widget.config.enableFullscreen : true; - this.hasTimewindow = (this.widget.type === widgetType.timeseries || this.widget.type === widgetType.alarm) ? + let canHaveTimewindow = false; + let onlyQuickInterval = false; + let onlyHistoryTimewindow = false; + if (this.widget.type === widgetType.timeseries || this.widget.type === widgetType.alarm) { + canHaveTimewindow = true; + } else if (this.widget.type === widgetType.latest) { + canHaveTimewindow = datasourcesHasAggregation(this.widget.config.datasources); + onlyQuickInterval = canHaveTimewindow; + if (canHaveTimewindow) { + onlyHistoryTimewindow = datasourcesHasOnlyComparisonAggregation(this.widget.config.datasources); + } + } + + this.hasTimewindow = canHaveTimewindow ? (isDefined(this.widget.config.useDashboardTimewindow) ? (!this.widget.config.useDashboardTimewindow && (isUndefined(this.widget.config.displayTimewindow) || this.widget.config.displayTimewindow)) : false) : false; + this.onlyQuickInterval = onlyQuickInterval; + this.onlyHistoryTimewindow = onlyHistoryTimewindow; + this.hasAggregation = this.widget.type === widgetType.timeseries; this.style = { diff --git a/ui-ngx/src/app/shared/components/time/quick-time-interval.component.scss b/ui-ngx/src/app/shared/components/time/quick-time-interval.component.scss new file mode 100644 index 0000000000..aeb3798f35 --- /dev/null +++ b/ui-ngx/src/app/shared/components/time/quick-time-interval.component.scss @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import '../../../../scss/constants'; + +:host { + min-width: 355px; + + @media #{$mat-xs} { + min-width: 0; + width: 100%; + } +} diff --git a/ui-ngx/src/app/shared/components/time/quick-time-interval.component.ts b/ui-ngx/src/app/shared/components/time/quick-time-interval.component.ts index c318bc4dfb..c8167dbc0e 100644 --- a/ui-ngx/src/app/shared/components/time/quick-time-interval.component.ts +++ b/ui-ngx/src/app/shared/components/time/quick-time-interval.component.ts @@ -21,6 +21,7 @@ import { QuickTimeInterval, QuickTimeIntervalTranslationMap } from '@shared/mode @Component({ selector: 'tb-quick-time-interval', templateUrl: './quick-time-interval.component.html', + styleUrls: ['./quick-time-interval.component.scss'], providers: [ { provide: NG_VALUE_ACCESSOR, diff --git a/ui-ngx/src/app/shared/components/time/timewindow-panel.component.html b/ui-ngx/src/app/shared/components/time/timewindow-panel.component.html index 1bb893d219..be9c38897b 100644 --- a/ui-ngx/src/app/shared/components/time/timewindow-panel.component.html +++ b/ui-ngx/src/app/shared/components/time/timewindow-panel.component.html @@ -29,32 +29,57 @@
- + -
- timewindow.last - +
+
+ + +
+
+ timewindow.last + +
-
- timewindow.interval - +
+
+ + +
+
+ timewindow.interval + +
+ +
diff --git a/ui-ngx/src/app/shared/components/time/timewindow-panel.component.ts b/ui-ngx/src/app/shared/components/time/timewindow-panel.component.ts index 09fa20aaa3..67ee2ac72b 100644 --- a/ui-ngx/src/app/shared/components/time/timewindow-panel.component.ts +++ b/ui-ngx/src/app/shared/components/time/timewindow-panel.component.ts @@ -36,6 +36,7 @@ export const TIMEWINDOW_PANEL_DATA = new InjectionToken('TimewindowPanelDat export interface TimewindowPanelData { historyOnly: boolean; + quickIntervalOnly: boolean; timewindow: Timewindow; aggregation: boolean; timezone: boolean; @@ -51,6 +52,8 @@ export class TimewindowPanelComponent extends PageComponent implements OnInit { historyOnly = false; + quickIntervalOnly = false; + aggregation = false; timezone = false; @@ -83,6 +86,7 @@ export class TimewindowPanelComponent extends PageComponent implements OnInit { public viewContainerRef: ViewContainerRef) { super(store); this.historyOnly = data.historyOnly; + this.quickIntervalOnly = data.quickIntervalOnly; this.timewindow = data.timewindow; this.aggregation = data.aggregation; this.timezone = data.timezone; @@ -91,6 +95,8 @@ export class TimewindowPanelComponent extends PageComponent implements OnInit { ngOnInit(): void { const hideInterval = this.timewindow.hideInterval || false; + const hideLastInterval = this.timewindow.hideLastInterval || false; + const hideQuickInterval = this.timewindow.hideQuickInterval || false; const hideAggregation = this.timewindow.hideAggregation || false; const hideAggInterval = this.timewindow.hideAggInterval || false; const hideTimezone = this.timewindow.hideTimezone || false; @@ -103,10 +109,11 @@ export class TimewindowPanelComponent extends PageComponent implements OnInit { ? this.timewindow.realtime.realtimeType : RealtimeWindowType.LAST_INTERVAL, disabled: hideInterval }), - timewindowMs: [ - this.timewindow.realtime && typeof this.timewindow.realtime.timewindowMs !== 'undefined' - ? this.timewindow.realtime.timewindowMs : null - ], + timewindowMs: this.fb.control({ + value: this.timewindow.realtime && typeof this.timewindow.realtime.timewindowMs !== 'undefined' + ? this.timewindow.realtime.timewindowMs : null, + disabled: hideInterval || hideLastInterval + }), interval: [ this.timewindow.realtime && typeof this.timewindow.realtime.interval !== 'undefined' ? this.timewindow.realtime.interval : null @@ -114,7 +121,7 @@ export class TimewindowPanelComponent extends PageComponent implements OnInit { quickInterval: this.fb.control({ value: this.timewindow.realtime && typeof this.timewindow.realtime.quickInterval !== 'undefined' ? this.timewindow.realtime.quickInterval : null, - disabled: hideInterval + disabled: hideInterval || hideQuickInterval }) } ), @@ -289,8 +296,40 @@ export class TimewindowPanelComponent extends PageComponent implements OnInit { this.timewindowForm.get('history.fixedTimewindow').enable({emitEvent: false}); this.timewindowForm.get('history.quickInterval').enable({emitEvent: false}); this.timewindowForm.get('realtime.realtimeType').enable({emitEvent: false}); - this.timewindowForm.get('realtime.timewindowMs').enable({emitEvent: false}); - this.timewindowForm.get('realtime.quickInterval').enable({emitEvent: false}); + if (!this.timewindow.hideLastInterval) { + this.timewindowForm.get('realtime.timewindowMs').enable({emitEvent: false}); + } + if (!this.timewindow.hideQuickInterval) { + this.timewindowForm.get('realtime.quickInterval').enable({emitEvent: false}); + } + } + this.timewindowForm.markAsDirty(); + } + + onHideLastIntervalChanged() { + if (this.timewindow.hideLastInterval) { + this.timewindowForm.get('realtime.timewindowMs').disable({emitEvent: false}); + if (!this.timewindow.hideQuickInterval) { + this.timewindowForm.get('realtime.realtimeType').setValue(RealtimeWindowType.INTERVAL); + } + } else { + if (!this.timewindow.hideInterval) { + this.timewindowForm.get('realtime.timewindowMs').enable({emitEvent: false}); + } + } + this.timewindowForm.markAsDirty(); + } + + onHideQuickIntervalChanged() { + if (this.timewindow.hideQuickInterval) { + this.timewindowForm.get('realtime.quickInterval').disable({emitEvent: false}); + if (!this.timewindow.hideLastInterval) { + this.timewindowForm.get('realtime.realtimeType').setValue(RealtimeWindowType.LAST_INTERVAL); + } + } else { + if (!this.timewindow.hideInterval) { + this.timewindowForm.get('realtime.quickInterval').enable({emitEvent: false}); + } } this.timewindowForm.markAsDirty(); } diff --git a/ui-ngx/src/app/shared/components/time/timewindow.component.ts b/ui-ngx/src/app/shared/components/time/timewindow.component.ts index 6f34131241..9b6127181e 100644 --- a/ui-ngx/src/app/shared/components/time/timewindow.component.ts +++ b/ui-ngx/src/app/shared/components/time/timewindow.component.ts @@ -75,13 +75,41 @@ export class TimewindowComponent implements OnInit, OnDestroy, ControlValueAcces @Input() set historyOnly(val) { - this.historyOnlyValue = coerceBooleanProperty(val); + const newHistoryOnlyValue = coerceBooleanProperty(val); + if (this.historyOnlyValue !== newHistoryOnlyValue) { + this.historyOnlyValue = newHistoryOnlyValue; + if (this.onHistoryOnlyChanged()) { + this.notifyChanged(); + } + } } get historyOnly() { return this.historyOnlyValue; } + alwaysDisplayTypePrefixValue = false; + + @Input() + set alwaysDisplayTypePrefix(val) { + this.alwaysDisplayTypePrefixValue = coerceBooleanProperty(val); + } + + get alwaysDisplayTypePrefix() { + return this.alwaysDisplayTypePrefixValue; + } + + quickIntervalOnlyValue = false; + + @Input() + set quickIntervalOnly(val) { + this.quickIntervalOnlyValue = coerceBooleanProperty(val); + } + + get quickIntervalOnly() { + return this.quickIntervalOnlyValue; + } + aggregationValue = false; @Input() @@ -240,6 +268,7 @@ export class TimewindowComponent implements OnInit, OnDestroy, ControlValueAcces { timewindow: deepClone(this.innerValue), historyOnly: this.historyOnly, + quickIntervalOnly: this.quickIntervalOnly, aggregation: this.aggregation, timezone: this.timezone, isEdit: this.isEdit @@ -265,6 +294,17 @@ export class TimewindowComponent implements OnInit, OnDestroy, ControlValueAcces return Injector.create({parent: this.viewContainerRef.injector, providers}); } + private onHistoryOnlyChanged(): boolean { + if (this.historyOnlyValue && this.innerValue) { + if (this.innerValue.selectedTab !== TimewindowType.HISTORY) { + this.innerValue.selectedTab = TimewindowType.HISTORY; + this.updateDisplayValue(); + return true; + } + } + return false; + } + registerOnChange(fn: any): void { this.propagateChange = fn; } @@ -278,9 +318,15 @@ export class TimewindowComponent implements OnInit, OnDestroy, ControlValueAcces } writeValue(obj: Timewindow): void { - this.innerValue = initModelFromDefaultTimewindow(obj, this.timeService); + this.innerValue = initModelFromDefaultTimewindow(obj, this.quickIntervalOnly, this.timeService); this.timewindowDisabled = this.isTimewindowDisabled(); - this.updateDisplayValue(); + if (this.onHistoryOnlyChanged()) { + setTimeout(() => { + this.notifyChanged(); + }); + } else { + this.updateDisplayValue(); + } } notifyChanged() { @@ -297,7 +343,7 @@ export class TimewindowComponent implements OnInit, OnDestroy, ControlValueAcces this.millisecondsToTimeStringPipe.transform(this.innerValue.realtime.timewindowMs); } } else { - this.innerValue.displayValue = !this.historyOnly ? (this.translate.instant('timewindow.history') + ' - ') : ''; + this.innerValue.displayValue = (!this.historyOnly || this.alwaysDisplayTypePrefix) ? (this.translate.instant('timewindow.history') + ' - ') : ''; if (this.innerValue.history.historyType === HistoryWindowType.LAST_INTERVAL) { this.innerValue.displayValue += this.translate.instant('timewindow.last-prefix') + ' ' + this.millisecondsToTimeStringPipe.transform(this.innerValue.history.timewindowMs); diff --git a/ui-ngx/src/app/shared/models/query/query.models.ts b/ui-ngx/src/app/shared/models/query/query.models.ts index 81ab569a88..c1bf255cd3 100644 --- a/ui-ngx/src/app/shared/models/query/query.models.ts +++ b/ui-ngx/src/app/shared/models/query/query.models.ts @@ -738,7 +738,6 @@ export function createDefaultEntityDataPageLink(pageSize: number): EntityDataPag } export const singleEntityDataPageLink: EntityDataPageLink = createDefaultEntityDataPageLink(1); -export const defaultEntityDataPageLink: EntityDataPageLink = createDefaultEntityDataPageLink(1024); export interface EntityCountQuery { entityFilter: EntityFilter; @@ -761,12 +760,19 @@ export interface AlarmDataQuery extends AbstractDataQuery { export interface TsValue { ts: number; value: string; + count?: number; +} + +export interface ComparisonTsValue { + current?: TsValue; + previous?: TsValue; } export interface EntityData { entityId: EntityId; latest: {[entityKeyType: string]: {[key: string]: TsValue}}; timeseries: {[key: string]: Array}; + aggLatest?: {[id: number]: ComparisonTsValue}; } export interface AlarmData extends AlarmInfo { diff --git a/ui-ngx/src/app/shared/models/telemetry/telemetry.models.ts b/ui-ngx/src/app/shared/models/telemetry/telemetry.models.ts index 4b8c21b891..5fef4b832e 100644 --- a/ui-ngx/src/app/shared/models/telemetry/telemetry.models.ts +++ b/ui-ngx/src/app/shared/models/telemetry/telemetry.models.ts @@ -173,12 +173,35 @@ export interface TimeSeriesCmd { fetchLatestPreviousPoint?: boolean; } +export interface AggKey { + id: number; + key: string; + agg: AggregationType; + previousStartTs?: number; + previousEndTs?: number; + previousValueOnly?: boolean; +} + +export interface AggEntityHistoryCmd { + keys: Array; + startTs: number; + endTs: number; +} + +export interface AggTimeSeriesCmd { + keys: Array; + startTs: number; + timeWindow: number; +} + export class EntityDataCmd implements WebsocketCmd { cmdId: number; query?: EntityDataQuery; historyCmd?: EntityHistoryCmd; latestCmd?: LatestValueCmd; tsCmd?: TimeSeriesCmd; + aggHistoryCmd?: AggEntityHistoryCmd; + aggTsCmd?: AggTimeSeriesCmd; public isEmpty(): boolean { return !this.query && !this.historyCmd && !this.latestCmd && !this.tsCmd; @@ -212,15 +235,6 @@ export class AlarmDataUnsubscribeCmd implements WebsocketCmd { } export class TelemetryPluginCmdsWrapper { - attrSubCmds: Array; - tsSubCmds: Array; - historyCmds: Array; - entityDataCmds: Array; - entityDataUnsubscribeCmds: Array; - alarmDataCmds: Array; - alarmDataUnsubscribeCmds: Array; - entityCountCmds: Array; - entityCountUnsubscribeCmds: Array; constructor() { this.attrSubCmds = []; @@ -233,6 +247,24 @@ export class TelemetryPluginCmdsWrapper { this.entityCountCmds = []; this.entityCountUnsubscribeCmds = []; } + attrSubCmds: Array; + tsSubCmds: Array; + historyCmds: Array; + entityDataCmds: Array; + entityDataUnsubscribeCmds: Array; + alarmDataCmds: Array; + alarmDataUnsubscribeCmds: Array; + entityCountCmds: Array; + entityCountUnsubscribeCmds: Array; + + private static popCmds(cmds: Array, leftCount: number): Array { + const toPublish = Math.min(cmds.length, leftCount); + if (toPublish > 0) { + return cmds.splice(0, toPublish); + } else { + return []; + } + } public hasCommands(): boolean { return this.tsSubCmds.length > 0 || @@ -261,38 +293,33 @@ export class TelemetryPluginCmdsWrapper { public preparePublishCommands(maxCommands: number): TelemetryPluginCmdsWrapper { const preparedWrapper = new TelemetryPluginCmdsWrapper(); let leftCount = maxCommands; - preparedWrapper.tsSubCmds = this.popCmds(this.tsSubCmds, leftCount); + preparedWrapper.tsSubCmds = TelemetryPluginCmdsWrapper.popCmds(this.tsSubCmds, leftCount); leftCount -= preparedWrapper.tsSubCmds.length; - preparedWrapper.historyCmds = this.popCmds(this.historyCmds, leftCount); + preparedWrapper.historyCmds = TelemetryPluginCmdsWrapper.popCmds(this.historyCmds, leftCount); leftCount -= preparedWrapper.historyCmds.length; - preparedWrapper.attrSubCmds = this.popCmds(this.attrSubCmds, leftCount); + preparedWrapper.attrSubCmds = TelemetryPluginCmdsWrapper.popCmds(this.attrSubCmds, leftCount); leftCount -= preparedWrapper.attrSubCmds.length; - preparedWrapper.entityDataCmds = this.popCmds(this.entityDataCmds, leftCount); + preparedWrapper.entityDataCmds = TelemetryPluginCmdsWrapper.popCmds(this.entityDataCmds, leftCount); leftCount -= preparedWrapper.entityDataCmds.length; - preparedWrapper.entityDataUnsubscribeCmds = this.popCmds(this.entityDataUnsubscribeCmds, leftCount); + preparedWrapper.entityDataUnsubscribeCmds = TelemetryPluginCmdsWrapper.popCmds(this.entityDataUnsubscribeCmds, leftCount); leftCount -= preparedWrapper.entityDataUnsubscribeCmds.length; - preparedWrapper.alarmDataCmds = this.popCmds(this.alarmDataCmds, leftCount); + preparedWrapper.alarmDataCmds = TelemetryPluginCmdsWrapper.popCmds(this.alarmDataCmds, leftCount); leftCount -= preparedWrapper.alarmDataCmds.length; - preparedWrapper.alarmDataUnsubscribeCmds = this.popCmds(this.alarmDataUnsubscribeCmds, leftCount); + preparedWrapper.alarmDataUnsubscribeCmds = TelemetryPluginCmdsWrapper.popCmds(this.alarmDataUnsubscribeCmds, leftCount); leftCount -= preparedWrapper.alarmDataUnsubscribeCmds.length; - preparedWrapper.entityCountCmds = this.popCmds(this.entityCountCmds, leftCount); + preparedWrapper.entityCountCmds = TelemetryPluginCmdsWrapper.popCmds(this.entityCountCmds, leftCount); leftCount -= preparedWrapper.entityCountCmds.length; - preparedWrapper.entityCountUnsubscribeCmds = this.popCmds(this.entityCountUnsubscribeCmds, leftCount); + preparedWrapper.entityCountUnsubscribeCmds = TelemetryPluginCmdsWrapper.popCmds(this.entityCountUnsubscribeCmds, leftCount); return preparedWrapper; } - - private popCmds(cmds: Array, leftCount: number): Array { - const toPublish = Math.min(cmds.length, leftCount); - if (toPublish > 0) { - return cmds.splice(0, toPublish); - } else { - return []; - } - } } export interface SubscriptionData { - [key: string]: [number, any][]; + [key: string]: [number, any, number?][]; +} + +export interface IndexedSubscriptionData { + [id: number]: [number, any, number?][]; } export interface SubscriptionDataHolder { @@ -434,16 +461,7 @@ export class EntityDataUpdate extends DataUpdate { super(msg); } - public prepareData(tsOffset: number) { - if (this.data) { - this.processEntityData(this.data.data, tsOffset); - } - if (this.update) { - this.processEntityData(this.update, tsOffset); - } - } - - private processEntityData(data: Array, tsOffset: number) { + private static processEntityData(data: Array, tsOffset: number) { for (const entityData of data) { if (entityData.timeseries) { for (const key of Object.keys(entityData.timeseries)) { @@ -471,28 +489,28 @@ export class EntityDataUpdate extends DataUpdate { } } } + + public prepareData(tsOffset: number) { + if (this.data) { + EntityDataUpdate.processEntityData(this.data.data, tsOffset); + } + if (this.update) { + EntityDataUpdate.processEntityData(this.update, tsOffset); + } + } } export class AlarmDataUpdate extends DataUpdate { - allowedEntities: number; - totalEntities: number; constructor(msg: AlarmDataUpdateMsg) { super(msg); this.allowedEntities = msg.allowedEntities; this.totalEntities = msg.totalEntities; } + allowedEntities: number; + totalEntities: number; - public prepareData(tsOffset: number) { - if (this.data) { - this.processAlarmData(this.data.data, tsOffset); - } - if (this.update) { - this.processAlarmData(this.update, tsOffset); - } - } - - private processAlarmData(data: Array, tsOffset: number) { + private static processAlarmData(data: Array, tsOffset: number) { for (const alarmData of data) { alarmData.createdTime += tsOffset; if (alarmData.ackTs) { @@ -524,6 +542,15 @@ export class AlarmDataUpdate extends DataUpdate { } } } + + public prepareData(tsOffset: number) { + if (this.data) { + AlarmDataUpdate.processAlarmData(this.data.data, tsOffset); + } + if (this.update) { + AlarmDataUpdate.processAlarmData(this.update, tsOffset); + } + } } export class EntityCountUpdate extends CmdUpdate { diff --git a/ui-ngx/src/app/shared/models/time/time.models.ts b/ui-ngx/src/app/shared/models/time/time.models.ts index 2f90d39ca1..d657d5cda4 100644 --- a/ui-ngx/src/app/shared/models/time/time.models.ts +++ b/ui-ngx/src/app/shared/models/time/time.models.ts @@ -96,6 +96,8 @@ export interface Timewindow { displayValue?: string; displayTimezoneAbbr?: string; hideInterval?: boolean; + hideQuickInterval?: boolean; + hideLastInterval?: boolean; hideAggregation?: boolean; hideAggInterval?: boolean; hideTimezone?: boolean; @@ -188,6 +190,8 @@ export function defaultTimewindow(timeService: TimeService): Timewindow { return { displayValue: '', hideInterval: false, + hideLastInterval: false, + hideQuickInterval: false, hideAggregation: false, hideAggInterval: false, hideTimezone: false, @@ -223,10 +227,12 @@ function getTimewindowType(timewindow: Timewindow): TimewindowType { } } -export function initModelFromDefaultTimewindow(value: Timewindow, timeService: TimeService): Timewindow { +export function initModelFromDefaultTimewindow(value: Timewindow, quickIntervalOnly: boolean, timeService: TimeService): Timewindow { const model = defaultTimewindow(timeService); if (value) { model.hideInterval = value.hideInterval; + model.hideLastInterval = value.hideLastInterval; + model.hideQuickInterval = value.hideQuickInterval; model.hideAggregation = value.hideAggregation; model.hideAggInterval = value.hideAggInterval; model.hideTimezone = value.hideTimezone; @@ -281,6 +287,9 @@ export function initModelFromDefaultTimewindow(value: Timewindow, timeService: T } model.timezone = value.timezone; } + if (quickIntervalOnly) { + model.realtime.realtimeType = RealtimeWindowType.INTERVAL; + } return model; } @@ -304,6 +313,8 @@ export function toHistoryTimewindow(timewindow: Timewindow, startTimeMs: number, } return { hideInterval: timewindow.hideInterval || false, + hideLastInterval: timewindow.hideLastInterval || false, + hideQuickInterval: timewindow.hideQuickInterval || false, hideAggregation: timewindow.hideAggregation || false, hideAggInterval: timewindow.hideAggInterval || false, hideTimezone: timewindow.hideTimezone || false, @@ -694,6 +705,8 @@ export function createTimewindowForComparison(subscriptionTimewindow: Subscripti export function cloneSelectedTimewindow(timewindow: Timewindow): Timewindow { const cloned: Timewindow = {}; cloned.hideInterval = timewindow.hideInterval || false; + cloned.hideLastInterval = timewindow.hideLastInterval || false; + cloned.hideQuickInterval = timewindow.hideQuickInterval || false; cloned.hideAggregation = timewindow.hideAggregation || false; cloned.hideAggInterval = timewindow.hideAggInterval || false; cloned.hideTimezone = timewindow.hideTimezone || false; diff --git a/ui-ngx/src/app/shared/models/widget.models.ts b/ui-ngx/src/app/shared/models/widget.models.ts index f8991f55c7..92b977e346 100644 --- a/ui-ngx/src/app/shared/models/widget.models.ts +++ b/ui-ngx/src/app/shared/models/widget.models.ts @@ -17,7 +17,7 @@ import { BaseData } from '@shared/models/base-data'; import { TenantId } from '@shared/models/id/tenant-id'; import { WidgetTypeId } from '@shared/models/id/widget-type-id'; -import { Timewindow } from '@shared/models/time/time.models'; +import { AggregationType, ComparisonDuration, Timewindow } from '@shared/models/time/time.models'; import { EntityType } from '@shared/models/entity-type.models'; import { AlarmSearchStatus, AlarmSeverity } from '@shared/models/alarm.models'; import { DataKeyType } from './telemetry/telemetry.models'; @@ -259,8 +259,27 @@ export function defaultLegendConfig(wType: widgetType): LegendConfig { }; } +export enum ComparisonResultType { + PREVIOUS_VALUE = 'PREVIOUS_VALUE', + DELTA_ABSOLUTE = 'DELTA_ABSOLUTE', + DELTA_PERCENT = 'DELTA_PERCENT' +} + +export const comparisonResultTypeTranslationMap = new Map( + [ + [ComparisonResultType.PREVIOUS_VALUE, 'datakey.delta-calculation-result-previous-value'], + [ComparisonResultType.DELTA_ABSOLUTE, 'datakey.delta-calculation-result-delta-absolute'], + [ComparisonResultType.DELTA_PERCENT, 'datakey.delta-calculation-result-delta-percent'] + ] +); + export interface KeyInfo { name: string; + aggregationType?: AggregationType; + comparisonEnabled?: boolean; + timeForComparison?: ComparisonDuration; + comparisonCustomIntervalValue?: number; + comparisonResultType?: ComparisonResultType; label?: string; color?: string; funcBody?: string; @@ -269,6 +288,18 @@ export interface KeyInfo { decimals?: number; } +export const dataKeyAggregationTypeHintTranslationMap = new Map( + [ + [AggregationType.MIN, 'datakey.aggregation-type-min-hint'], + [AggregationType.MAX, 'datakey.aggregation-type-max-hint'], + [AggregationType.AVG, 'datakey.aggregation-type-avg-hint'], + [AggregationType.SUM, 'datakey.aggregation-type-sum-hint'], + [AggregationType.COUNT, 'datakey.aggregation-type-count-hint'], + [AggregationType.NONE, 'datakey.aggregation-type-none-hint'], + ] +); + + export interface DataKey extends KeyInfo { type: DataKeyType; pattern?: string; @@ -322,6 +353,37 @@ export interface Datasource { [key: string]: any; } +export function datasourcesHasAggregation(datasources?: Array): boolean { + if (datasources) { + const foundDatasource = datasources.find(datasource => { + const found = datasource.dataKeys && datasource.dataKeys.find(key => key.type === DataKeyType.timeseries && + key.aggregationType && key.aggregationType !== AggregationType.NONE); + return !!found; + }); + if (foundDatasource) { + return true; + } + } + return false; +} + +export function datasourcesHasOnlyComparisonAggregation(datasources?: Array): boolean { + if (!datasourcesHasAggregation(datasources)) { + return false; + } + if (datasources) { + const foundDatasource = datasources.find(datasource => { + const found = datasource.dataKeys && datasource.dataKeys.find(key => key.type === DataKeyType.timeseries && + key.aggregationType && key.aggregationType !== AggregationType.NONE && !key.comparisonEnabled); + return !!found; + }); + if (foundDatasource) { + return false; + } + } + return true; +} + export interface FormattedData { $datasource: Datasource; entityName: string; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 26f6834b11..1fd934d92d 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1039,7 +1039,22 @@ "value-description": "the current value;", "prev-value-description": "result of the previous function call;", "time-prev-description": "timestamp of the previous value;", - "prev-orig-value-description": "original previous value;" + "prev-orig-value-description": "original previous value;", + "aggregation": "Aggregation", + "aggregation-type-hint-common": "For performance reasons, the aggregated values calculation is available only for fixed time intervals like \"current day\", \"current month\", etc, and is not available for sliding window intervals like 'last 30 minutes' or 'last 24 hours'.", + "aggregation-type-none-hint": "Take latest value.", + "aggregation-type-min-hint": "Find minimum value among data points within a selected time window.", + "aggregation-type-max-hint": "Find maximum value among data points within a selected time window.", + "aggregation-type-avg-hint": "Calculate an average value among data points within a selected time window.", + "aggregation-type-sum-hint": "Sum all values of the data points within a selected time window.", + "aggregation-type-count-hint": "Total number of the data points within a selected time window.", + "delta-calculation": "Delta calculation", + "enable-delta-calculation": "Enable delta calculation", + "enable-delta-calculation-hint": "When enabled, the data key value is calculated based on the aggregated values for a selected time window and a specified comparison period. For performance reasons, the delta calculation is available only for history time windows and not for real-time values. For example, you may calculate the delta between the energy consumption for yesterday compared to the energy consumption for the day before yesterday.", + "delta-calculation-result": "Delta calculation result", + "delta-calculation-result-previous-value": "Previous value", + "delta-calculation-result-delta-absolute": "Delta (absolute)", + "delta-calculation-result-delta-percent": "Delta (percent)" }, "datasource": { "type": "Datasource type",