From e3041adc42d71a3751823b42113be69eaef02299 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 9 Jan 2024 18:18:10 +0200 Subject: [PATCH 1/8] Grouping interval structures --- .../server/ThingsboardServerApplication.java | 6 ++++- .../ws/telemetry/cmd/v2/EntityHistoryCmd.java | 3 +++ .../service/ws/telemetry/cmd/v2/GetTsCmd.java | 5 +++++ .../ws/telemetry/cmd/v2/TimeSeriesCmd.java | 3 +++ .../server/common/data/kv/IntervalType.java | 22 +++++++++++++++++++ 5 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/kv/IntervalType.java diff --git a/application/src/main/java/org/thingsboard/server/ThingsboardServerApplication.java b/application/src/main/java/org/thingsboard/server/ThingsboardServerApplication.java index 60f69cd8a7..77442e53a8 100644 --- a/application/src/main/java/org/thingsboard/server/ThingsboardServerApplication.java +++ b/application/src/main/java/org/thingsboard/server/ThingsboardServerApplication.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; +import java.time.ZoneId; import java.util.Arrays; @SpringBootConfiguration @@ -33,7 +34,10 @@ public class ThingsboardServerApplication { private static final String DEFAULT_SPRING_CONFIG_PARAM = SPRING_CONFIG_NAME_KEY + "=" + "thingsboard"; public static void main(String[] args) { - SpringApplication.run(ThingsboardServerApplication.class, updateArguments(args)); + + ZoneId.getAvailableZoneIds().stream().sorted().forEach(System.out::println); + +// SpringApplication.run(ThingsboardServerApplication.class, updateArguments(args)); } private static String[] updateArguments(String[] args) { diff --git a/application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/EntityHistoryCmd.java b/application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/EntityHistoryCmd.java index 6ae9a03ff8..3b3dd0d55d 100644 --- a/application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/EntityHistoryCmd.java +++ b/application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/EntityHistoryCmd.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.ws.telemetry.cmd.v2; import lombok.Data; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.IntervalType; import java.util.List; @@ -26,7 +27,9 @@ public class EntityHistoryCmd implements GetTsCmd { private List keys; private long startTs; private long endTs; + private IntervalType intervalType; private long interval; + private String timeZoneId; private int limit; private Aggregation agg; private boolean fetchLatestPreviousPoint; diff --git a/application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/GetTsCmd.java b/application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/GetTsCmd.java index 039873bb18..80a749fbe7 100644 --- a/application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/GetTsCmd.java +++ b/application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/GetTsCmd.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.ws.telemetry.cmd.v2; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.IntervalType; import java.util.List; @@ -27,8 +28,12 @@ public interface GetTsCmd { List getKeys(); + IntervalType getIntervalType(); + long getInterval(); + String getTimeZoneId(); + int getLimit(); Aggregation getAgg(); diff --git a/application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/TimeSeriesCmd.java b/application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/TimeSeriesCmd.java index e7a9e03d17..3c3d53e426 100644 --- a/application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/TimeSeriesCmd.java +++ b/application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/TimeSeriesCmd.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.ws.telemetry.cmd.v2; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.IntervalType; import java.util.List; @@ -27,7 +28,9 @@ public class TimeSeriesCmd implements GetTsCmd { private List keys; private long startTs; private long timeWindow; + private IntervalType intervalType; private long interval; + private String timeZoneId; private int limit; private Aggregation agg; private boolean fetchLatestPreviousPoint; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/IntervalType.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/IntervalType.java new file mode 100644 index 0000000000..e5ebb7f1a1 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/IntervalType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.kv; + +public enum IntervalType { + + MILLISECONDS, WEEK, WEEK_ISO, MONTH, QUARTER + +} From 7fc22311898196f5758630a80bbf33b029c47c3c Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 11 Jan 2024 12:40:50 +0200 Subject: [PATCH 2/8] Interval types back-end implementation --- .../controller/TelemetryController.java | 21 ++++- ...efaultTbEntityDataSubscriptionService.java | 7 +- .../service/ws/telemetry/cmd/v2/GetTsCmd.java | 13 +++ .../controller/TelemetryControllerTest.java | 59 ++++++++++++ .../common/data/kv/AggregationParams.java | 92 +++++++++++++++++++ .../common/data/kv/BaseReadTsKvQuery.java | 26 ++++-- .../server/common/data/kv/IntervalType.java | 2 +- .../server/common/data/kv/ReadTsKvQuery.java | 12 ++- ...stractChunkedAggregationTimeseriesDao.java | 23 ++++- .../timescale/TimescaleTimeseriesDao.java | 24 ++++- .../CassandraBaseTimeseriesDao.java | 18 +++- .../dao/timeseries/TsKvQueryCursor.java | 2 +- .../server/dao/util/TimeUtils.java | 45 +++++++++ .../server/dao/util/TimeUtilsTest.java | 60 ++++++++++++ 14 files changed, 370 insertions(+), 34 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/kv/AggregationParams.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/util/TimeUtils.java create mode 100644 dao/src/test/java/org/thingsboard/server/dao/util/TimeUtilsTest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index 295b8cad99..288828f046 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -45,6 +45,8 @@ import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.kv.AggregationParams; +import org.thingsboard.server.common.data.kv.IntervalType; import org.thingsboard.server.common.msg.rule.engine.DeviceAttributesEventNotificationMsg; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; @@ -310,8 +312,12 @@ public class TelemetryController extends BaseController { @RequestParam(name = "startTs") Long startTs, @ApiParam(value = "A long value representing the end timestamp of the time range in milliseconds, UTC.") @RequestParam(name = "endTs") Long endTs, + @ApiParam(value = "A string value representing the type fo the interval.", allowableValues = "MILLISECONDS, WEEK, WEEK_ISO, MONTH, QUARTER") + @RequestParam(name = "intervalType", required = false) IntervalType intervalType, @ApiParam(value = "A long value representing the aggregation interval range in milliseconds.") @RequestParam(name = "interval", defaultValue = "0") Long interval, + @ApiParam(value = "A string value representing the timezone that will be used to calculate exact timestamps for 'WEEK', 'WEEK_ISO', 'MONTH' and 'QUARTER' interval types.") + @RequestParam(name = "timeZone", required = false) String timeZone, @ApiParam(value = "An integer value that represents a max number of timeseries data points to fetch." + " This parameter is used only in the case if 'agg' parameter is set to 'NONE'.", defaultValue = "100") @RequestParam(name = "limit", defaultValue = "100") Integer limit, @@ -325,11 +331,16 @@ public class TelemetryController extends BaseController { @RequestParam(name = "useStrictDataTypes", required = false, defaultValue = "false") Boolean useStrictDataTypes) throws ThingsboardException { return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.READ_TELEMETRY, entityType, entityIdStr, (result, tenantId, entityId) -> { - // If interval is 0, convert this to a NONE aggregation, which is probably what the user really wanted - Aggregation agg = interval == 0L ? Aggregation.valueOf(Aggregation.NONE.name()) : Aggregation.valueOf(aggStr); - List queries = toKeysList(keys).stream().map(key -> new BaseReadTsKvQuery(key, startTs, endTs, interval, limit, agg, orderBy)) - .collect(Collectors.toList()); - + AggregationParams params; + Aggregation agg = Aggregation.valueOf(aggStr); + if (Aggregation.NONE.equals(agg)) { + params = AggregationParams.none(); + } else if (intervalType == null || IntervalType.MILLISECONDS.equals(intervalType)) { + params = interval == 0L ? AggregationParams.none() : AggregationParams.milliseconds(agg, interval); + } else { + params = AggregationParams.calendar(agg, intervalType, timeZone); + } + List queries = toKeysList(keys).stream().map(key -> new BaseReadTsKvQuery(key, startTs, endTs, params, limit, orderBy)).collect(Collectors.toList()); Futures.addCallback(tsService.findAll(tenantId, entityId, queries), getTsKvListCallback(result, useStrictDataTypes), MoreExecutors.directExecutor()); }); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java index 5251ee72f6..6e31a24275 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java @@ -560,17 +560,14 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc List keys = cmd.getKeys(); List finalTsKvQueryList; List tsKvQueryList = keys.stream().map(key -> { - var query = new BaseReadTsKvQuery( - key, cmd.getStartTs(), cmd.getEndTs(), cmd.getInterval(), getLimit(cmd.getLimit()), cmd.getAgg() - ); + var query = new BaseReadTsKvQuery(key, cmd.getStartTs(), cmd.getEndTs(), cmd.toAggregationParams(), getLimit(cmd.getLimit())); queriesKeys.put(query.getId(), query.getKey()); return query; }).collect(Collectors.toList()); if (cmd.isFetchLatestPreviousPoint()) { finalTsKvQueryList = new ArrayList<>(tsKvQueryList); finalTsKvQueryList.addAll(keys.stream().map(key -> { - var query = new BaseReadTsKvQuery( - key, cmd.getStartTs() - TimeUnit.DAYS.toMillis(365), cmd.getStartTs(), cmd.getInterval(), 1, cmd.getAgg()); + var query = new BaseReadTsKvQuery(key, cmd.getStartTs() - TimeUnit.DAYS.toMillis(365), cmd.getStartTs(), cmd.toAggregationParams(), 1); queriesKeys.put(query.getId(), query.getKey()); return query; } diff --git a/application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/GetTsCmd.java b/application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/GetTsCmd.java index 80a749fbe7..ef5d9efb96 100644 --- a/application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/GetTsCmd.java +++ b/application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/GetTsCmd.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.ws.telemetry.cmd.v2; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AggregationParams; import org.thingsboard.server.common.data.kv.IntervalType; import java.util.List; @@ -40,4 +41,16 @@ public interface GetTsCmd { boolean isFetchLatestPreviousPoint(); + default AggregationParams toAggregationParams() { + var agg = getAgg(); + var intervalType = getIntervalType(); + if (agg == null || Aggregation.NONE.equals(agg)) { + return AggregationParams.none(); + } else if (intervalType == null || IntervalType.MILLISECONDS.equals(intervalType)) { + return AggregationParams.milliseconds(agg, getInterval()); + } else { + return AggregationParams.calendar(agg, intervalType, getTimeZoneId()); + } + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/TelemetryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TelemetryControllerTest.java index 8061cf0eab..98fbffaee5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TelemetryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TelemetryControllerTest.java @@ -15,12 +15,17 @@ */ package org.thingsboard.server.controller; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.Assert; import org.junit.Test; import org.springframework.test.context.TestPropertySource; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.SaveDeviceWithCredentialsRequest; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.SingleEntityFilter; import org.thingsboard.server.common.data.security.DeviceCredentials; @@ -28,6 +33,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.List; +import java.util.concurrent.TimeUnit; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.common.data.query.EntityKeyType.TIME_SERIES; @@ -51,6 +57,59 @@ public class TelemetryControllerTest extends AbstractControllerTest { doPostAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + "/timeseries/smth", invalidRequestBody, String.class, status().isBadRequest()); } + @Test + public void testTelemetryRequests() throws Exception { + loginTenantAdmin(); + Device device = createDevice(); + + var startTs = 1704899727000L; // Wednesday, January 10 15:15:27 GMT + var endOfWeek1Ts = 1705269600000L; // Monday, January 15, 2024 0:00:00 GMT+02:00 + var endOfWeek2Ts = 1705874400000L; // Monday, January 22, 2024 0:00:00 GMT+02:00 + var endTs = endOfWeek2Ts + TimeUnit.DAYS.toMillis(1) + TimeUnit.HOURS.toMillis(1); // Monday, January 23, 2024 1:00:00 GMT+02:00 + + var firstIntervalTs = startTs + (endOfWeek1Ts - startTs) / 2; + var secondIntervalTs = endOfWeek1Ts + (endOfWeek2Ts - endOfWeek1Ts) / 2; + var thirdIntervalTs = endOfWeek2Ts + (endTs - endOfWeek2Ts) / 2; + + var middleOfTheInterval = startTs + (endTs - startTs) / 2; + + tsService.save(tenantId, device.getId(), new BasicTsKvEntry(1704899728000L, new LongDataEntry("t", 1L))); // Wednesday, January 10 15:15:28 GMT + tsService.save(tenantId, device.getId(), new BasicTsKvEntry(1704899729000L, new LongDataEntry("t", 3L))); // Wednesday, January 10 15:15:29 GMT + tsService.save(tenantId, device.getId(), new BasicTsKvEntry(endOfWeek1Ts + 1000, new LongDataEntry("t", 7L))); // Monday, January 15, 2024 0:00:01 GMT+02:00 + tsService.save(tenantId, device.getId(), new BasicTsKvEntry(endOfWeek2Ts + 1000, new LongDataEntry("t", 11L))); // Monday, January 22, 2024 0:00:01 GMT+02:00 + + ObjectNode result = doGetAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + + "/values/timeseries?keys=t&startTs={startTs}&endTs={endTs}&agg={agg}&intervalType={intervalType}&timeZone={timeZone}", + ObjectNode.class, startTs, endTs, "SUM", "WEEK_ISO", "Europe/Kyiv"); + Assert.assertNotNull(result); + Assert.assertNotNull(result.get("t")); + Assert.assertEquals(3, result.get("t").size()); + + var firstIntervalResult = result.get("t").get(0); + Assert.assertEquals(4L, firstIntervalResult.get("value").asLong()); + Assert.assertEquals(firstIntervalTs, firstIntervalResult.get("ts").asLong()); + + var secondIntervalResult = result.get("t").get(1); + Assert.assertEquals(7L, secondIntervalResult.get("value").asLong()); + Assert.assertEquals(secondIntervalTs, secondIntervalResult.get("ts").asLong()); + + var thirdIntervalResult = result.get("t").get(2); + Assert.assertEquals(11L, thirdIntervalResult.get("value").asLong()); + Assert.assertEquals(thirdIntervalTs, thirdIntervalResult.get("ts").asLong()); + + result = doGetAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + + "/values/timeseries?keys=t&startTs={startTs}&endTs={endTs}&agg={agg}&intervalType={intervalType}&timeZone={timeZone}", + ObjectNode.class, startTs, endTs, "SUM", "MONTH", "Europe/Kyiv"); + + Assert.assertNotNull(result); + Assert.assertNotNull(result.get("t")); + Assert.assertEquals(1, result.get("t").size()); + + var monthResult = result.get("t").get(0); + Assert.assertEquals(22L, monthResult.get("value").asLong()); + Assert.assertEquals(middleOfTheInterval, monthResult.get("ts").asLong()); + } + @Test public void testDeleteAllTelemetryWithLatest() throws Exception { loginTenantAdmin(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/AggregationParams.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/AggregationParams.java new file mode 100644 index 0000000000..9d7470b676 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/AggregationParams.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.kv; + +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; +import org.thingsboard.server.common.data.StringUtils; + +import java.time.DateTimeException; +import java.time.ZoneId; +import java.time.zone.ZoneRulesException; +import java.util.concurrent.TimeUnit; + +@AllArgsConstructor +@EqualsAndHashCode +@Slf4j +public class AggregationParams { + @Getter + private final Aggregation aggregation; + @Getter + private final IntervalType intervalType; + @Getter + private final ZoneId tzId; + + private final long interval; + + public static AggregationParams none() { + return new AggregationParams(Aggregation.NONE, null, null, 0L); + } + + public static AggregationParams milliseconds(Aggregation aggregationType, long aggregationIntervalMs) { + return new AggregationParams(aggregationType, IntervalType.MILLISECONDS, null, aggregationIntervalMs); + } + + public static AggregationParams calendar(Aggregation aggregationType, IntervalType intervalType, String tzIdStr) { + return calendar(aggregationType, intervalType, getZoneId(tzIdStr)); + } + + public static AggregationParams calendar(Aggregation aggregationType, IntervalType intervalType, ZoneId tzId) { + return new AggregationParams(aggregationType, intervalType, tzId, 0L); + } + + public static AggregationParams of(Aggregation aggregation, IntervalType intervalType, ZoneId tzId, long interval) { + return new AggregationParams(aggregation, intervalType, tzId, interval); + } + + public long getInterval() { + if (intervalType == null) { + return 0L; + } else { + switch (intervalType) { + case WEEK: + case WEEK_ISO: + return TimeUnit.DAYS.toMillis(7); + case MONTH: + return TimeUnit.DAYS.toMillis(30); + case QUARTER: + return TimeUnit.DAYS.toMillis(90); + default: + return interval; + } + } + } + + private static ZoneId getZoneId(String tzIdStr) { + if (StringUtils.isEmpty(tzIdStr)) { + return ZoneId.systemDefault(); + } + try { + return ZoneId.of(tzIdStr); + } catch (DateTimeException e) { + log.warn("[{}] Failed to convert the time zone. Fallback to default.", tzIdStr); + return ZoneId.systemDefault(); + } + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java index 54b3a15bd2..09d52e46a1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java @@ -18,41 +18,47 @@ package org.thingsboard.server.common.data.kv; import lombok.Data; import lombok.EqualsAndHashCode; +import java.time.ZoneId; + @Data @EqualsAndHashCode(callSuper = true) public class BaseReadTsKvQuery extends BaseTsKvQuery implements ReadTsKvQuery { - private final long interval; + private final AggregationParams aggParameters; private final int limit; - private final Aggregation aggregation; private final String order; public BaseReadTsKvQuery(String key, long startTs, long endTs, long interval, int limit, Aggregation aggregation) { this(key, startTs, endTs, interval, limit, aggregation, "DESC"); } - public BaseReadTsKvQuery(String key, long startTs, long endTs, long interval, int limit, Aggregation aggregation, String order) { + public BaseReadTsKvQuery(String key, long startTs, long endTs, long interval, int limit, Aggregation aggregation, String descOrder) { + this(key, startTs, endTs, AggregationParams.of(aggregation, IntervalType.MILLISECONDS, ZoneId.systemDefault(), interval), limit, descOrder); + } + + public BaseReadTsKvQuery(String key, long startTs, long endTs, AggregationParams parameters, int limit) { + this(key, startTs, endTs, parameters, limit, "DESC"); + } + + public BaseReadTsKvQuery(String key, long startTs, long endTs, AggregationParams parameters, int limit, String order) { super(key, startTs, endTs); - this.interval = interval; + this.aggParameters = parameters; this.limit = limit; - this.aggregation = aggregation; this.order = order; } public BaseReadTsKvQuery(String key, long startTs, long endTs) { - this(key, startTs, endTs, endTs - startTs, 1, Aggregation.AVG, "DESC"); + this(key, startTs, endTs, AggregationParams.milliseconds(Aggregation.AVG, endTs - startTs), 1, "DESC"); } public BaseReadTsKvQuery(String key, long startTs, long endTs, int limit, String order) { - this(key, startTs, endTs, endTs - startTs, limit, Aggregation.NONE, order); + this(key, startTs, endTs, AggregationParams.none(), limit, order); } public BaseReadTsKvQuery(ReadTsKvQuery query, long startTs, long endTs) { super(query.getId(), query.getKey(), startTs, endTs); - this.interval = query.getInterval(); + this.aggParameters = query.getAggParameters(); this.limit = query.getLimit(); - this.aggregation = query.getAggregation(); this.order = query.getOrder(); } - } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/IntervalType.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/IntervalType.java index e5ebb7f1a1..121b5993e0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/IntervalType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/IntervalType.java @@ -17,6 +17,6 @@ package org.thingsboard.server.common.data.kv; public enum IntervalType { - MILLISECONDS, WEEK, WEEK_ISO, MONTH, QUARTER + MILLISECONDS, WEEK/*Sunday-Saturday*/, WEEK_ISO/*Monday-Sunday*/, MONTH, QUARTER } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/ReadTsKvQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/ReadTsKvQuery.java index e979ec665a..776179967f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/ReadTsKvQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/ReadTsKvQuery.java @@ -17,11 +17,17 @@ package org.thingsboard.server.common.data.kv; public interface ReadTsKvQuery extends TsKvQuery { - long getInterval(); + AggregationParams getAggParameters(); - int getLimit(); + default long getInterval(){ + return getAggParameters().getInterval(); + } + + default Aggregation getAggregation() { + return getAggParameters().getAggregation(); + } - Aggregation getAggregation(); + int getLimit(); String getOrder(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 96e87e954e..6c22935b61 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.IntervalType; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -37,9 +38,18 @@ import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; import org.thingsboard.server.dao.sqlts.insert.InsertTsRepository; import org.thingsboard.server.dao.sqlts.ts.TsKvRepository; import org.thingsboard.server.dao.timeseries.TimeseriesDao; +import org.thingsboard.server.dao.util.TimeUtils; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.time.temporal.IsoFields; +import java.time.temporal.TemporalUnit; +import java.time.temporal.WeekFields; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -112,16 +122,23 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq @Override public ListenableFuture findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - if (query.getAggregation() == Aggregation.NONE) { + var aggParams = query.getAggParameters(); + if (Aggregation.NONE.equals(aggParams.getAggregation())) { return Futures.immediateFuture(findAllAsyncWithLimit(entityId, query)); } else { List>> futures = new ArrayList<>(); + var intervalType = aggParams.getIntervalType(); long startPeriod = query.getStartTs(); long endPeriod = Math.max(query.getStartTs() + 1, query.getEndTs()); - long step = query.getInterval(); while (startPeriod < endPeriod) { long startTs = startPeriod; - long endTs = Math.min(startPeriod + step, endPeriod); + long endTs; + if (IntervalType.MILLISECONDS.equals(intervalType)) { + endTs = startPeriod + aggParams.getInterval(); + } else { + endTs = TimeUtils.calculateIntervalEnd(startTs, intervalType, aggParams.getTzId()); + } + endTs = Math.min(endTs, endPeriod); long ts = startTs + (endTs - startTs) / 2; ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation()); futures.add(aggregateTsKvEntry); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 6f1d569730..bb0ce06026 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; @@ -28,6 +29,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.IntervalType; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -35,11 +37,13 @@ import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity; +import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; import org.thingsboard.server.dao.sqlts.AbstractSqlTimeseriesDao; import org.thingsboard.server.dao.sqlts.insert.InsertTsRepository; import org.thingsboard.server.dao.timeseries.TimeseriesDao; +import org.thingsboard.server.dao.util.TimeUtils; import org.thingsboard.server.dao.util.TimescaleDBTsDao; import javax.annotation.PostConstruct; @@ -143,14 +147,28 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements @Override public ListenableFuture findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + var aggParams = query.getAggParameters(); + var intervalType = aggParams.getIntervalType(); if (query.getAggregation() == Aggregation.NONE) { return Futures.immediateFuture(findAllAsyncWithLimit(entityId, query)); - } else { + } else if (IntervalType.MILLISECONDS.equals(intervalType)) { long startTs = query.getStartTs(); long endTs = Math.max(query.getStartTs() + 1, query.getEndTs()); long timeBucket = query.getInterval(); List> data = findAllAndAggregateAsync(entityId, query.getKey(), startTs, endTs, timeBucket, query.getAggregation()); return getReadTsKvQueryResultFuture(query, Futures.immediateFuture(data)); + } else { + //TODO: @dshvaika improve according to native capabilities of Timescale. + long startPeriod = query.getStartTs(); + long endPeriod = Math.max(query.getStartTs() + 1, query.getEndTs()); + List timescaleTsKvEntities = new ArrayList<>(); + while (startPeriod < endPeriod) { + long startTs = startPeriod; + long endTs = Math.min(TimeUtils.calculateIntervalEnd(startTs, intervalType, aggParams.getTzId()), endPeriod); + timescaleTsKvEntities.addAll(switchAggregation(query.getKey(), startTs, endTs, endTs - startTs, query.getAggregation(), entityId.getId())); + startPeriod = endTs; + } + return getReadTsKvQueryResultFuture(query, Futures.immediateFuture(toResultList(entityId, query.getKey(), timescaleTsKvEntities))); } } @@ -187,6 +205,10 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements timescaleTsKvEntities.addAll(switchAggregation(key, startTs + interval, endTs, remainingPart, aggregation, entityId.getId())); } + return toResultList(entityId, key, timescaleTsKvEntities); + } + + private static List> toResultList(EntityId entityId, String key, List timescaleTsKvEntities) { if (!CollectionUtils.isEmpty(timescaleTsKvEntities)) { List> result = new ArrayList<>(); timescaleTsKvEntities.forEach(entity -> { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 71541962ec..31e29434e8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -41,6 +41,7 @@ import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.IntervalType; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; @@ -51,6 +52,7 @@ import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.nosql.TbResultSetFuture; import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsDao; +import org.thingsboard.server.dao.util.TimeUtils; import javax.annotation.Nullable; import javax.annotation.PostConstruct; @@ -227,18 +229,24 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD @Override public ListenableFuture findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - if (query.getAggregation() == Aggregation.NONE) { + var aggParams = query.getAggParameters(); + if (Aggregation.NONE.equals(aggParams.getAggregation())) { return findAllAsyncWithLimit(tenantId, entityId, query); } else { long startPeriod = query.getStartTs(); long endPeriod = Math.max(query.getStartTs() + 1, query.getEndTs()); - long step = Math.max(query.getInterval(), MIN_AGGREGATION_STEP_MS); List>> futures = new ArrayList<>(); + var intervalType = aggParams.getIntervalType(); while (startPeriod < endPeriod) { long startTs = startPeriod; - long endTs = Math.min(startPeriod + step, endPeriod); - long ts = endTs - startTs; - ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); + long endTs; + if (IntervalType.MILLISECONDS.equals(intervalType)) { + endTs = startPeriod + Math.max(query.getInterval(), MIN_AGGREGATION_STEP_MS); + } else { + endTs = TimeUtils.calculateIntervalEnd(startTs, aggParams.getIntervalType(), aggParams.getTzId()); + } + endTs = Math.min(endTs, endPeriod); + ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, endTs - startTs, 1, query.getAggregation(), query.getOrder()); futures.add(findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs))); startPeriod = endTs; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsKvQueryCursor.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsKvQueryCursor.java index 9c302be53d..8ab9a2c4da 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsKvQueryCursor.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsKvQueryCursor.java @@ -33,7 +33,7 @@ public class TsKvQueryCursor extends QueryCursor { @Getter private final List data; @Getter - private String orderBy; + private final String orderBy; private int partitionIndex; private int currentLimit; diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/TimeUtils.java b/dao/src/main/java/org/thingsboard/server/dao/util/TimeUtils.java new file mode 100644 index 0000000000..3a768673b8 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/util/TimeUtils.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.util; + +import org.thingsboard.server.common.data.kv.IntervalType; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.time.temporal.IsoFields; +import java.time.temporal.WeekFields; + +public class TimeUtils { + + public static long calculateIntervalEnd(long startTs, IntervalType intervalType, ZoneId tzId) { + var startTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(startTs), tzId); + switch (intervalType) { + case WEEK: + return startTime.truncatedTo(ChronoUnit.DAYS).with(WeekFields.SUNDAY_START.dayOfWeek(), 1).plusDays(7).toInstant().toEpochMilli(); + case WEEK_ISO: + return startTime.truncatedTo(ChronoUnit.DAYS).with(WeekFields.ISO.dayOfWeek(), 1).plusDays(7).toInstant().toEpochMilli(); + case MONTH: + return startTime.truncatedTo(ChronoUnit.DAYS).withDayOfMonth(1).plusMonths(1).toInstant().toEpochMilli(); + case QUARTER: + return startTime.truncatedTo(ChronoUnit.DAYS).with(IsoFields.DAY_OF_QUARTER, 1).plusMonths(3).toInstant().toEpochMilli(); + default: + throw new RuntimeException("Not supported!"); + } + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/util/TimeUtilsTest.java b/dao/src/test/java/org/thingsboard/server/dao/util/TimeUtilsTest.java new file mode 100644 index 0000000000..a3df974236 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/util/TimeUtilsTest.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.util; + +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.kv.IntervalType; + +import java.time.ZoneId; + +import static org.assertj.core.api.Assertions.assertThat; + +class TimeUtilsTest { + + @Test + void testWeekEnd() { + long ts = 1704899727000L; // Wednesday, January 10 15:15:27 GMT + assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.WEEK, ZoneId.of("Europe/Kyiv"))).isEqualTo(1705183200000L); // Sunday, January 14, 2024 0:00:00 GMT+02:00 + assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.WEEK_ISO, ZoneId.of("Europe/Kyiv"))).isEqualTo(1705269600000L); // Monday, January 15, 2024 0:00:00 GMT+02:00 + + assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.WEEK, ZoneId.of("Europe/Amsterdam"))).isEqualTo(1705186800000L); // Sunday, January 14, 2024 0:00:00 GMT+01:00 + assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.WEEK_ISO, ZoneId.of("Europe/Amsterdam"))).isEqualTo(1705273200000L); // Monday, January 15, 2024 0:00:00 GMT+01:00 + + ts = 1704621600000L; // Sunday, January 7, 2024 12:00:00 GMT+02:00 + assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.WEEK, ZoneId.of("Europe/Kyiv"))).isEqualTo(1705183200000L); // Sunday, January 14, 2024 0:00:00 GMT+02:00 + assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.WEEK_ISO, ZoneId.of("Europe/Kyiv"))).isEqualTo(1704664800000L); // Monday, January 8, 2024 0:00:00 GMT+02:00 + } + + + @Test + void testMonthEnd() { + long ts = 1704899727000L; // Wednesday, January 10 15:15:27 GMT + assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.MONTH, ZoneId.of("Europe/Kyiv"))).isEqualTo(1706738400000L); // Thursday, February 1, 2024 0:00:00 GMT+02:00 + assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.MONTH, ZoneId.of("Europe/Amsterdam"))).isEqualTo(1706742000000L); // Monday, January 15, 2024 0:00:00 GMT+02:00 + } + + @Test + void testQuarterEnd() { + long ts = 1704899727000L; // Wednesday, January 10 15:15:27 GMT + assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.QUARTER, ZoneId.of("Europe/Kyiv"))).isEqualTo(1711918800000L); // Monday, April 1, 2024 0:00:00 GMT+03:00 DST + assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.QUARTER, ZoneId.of("Europe/Amsterdam"))).isEqualTo(1711922400000L); // Monday, April 1, 2024 1:00:00 GMT+03:00 DST + + ts = 1711929600000L; // Monday, April 1, 2024 3:00:00 GMT+03:00 + assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.QUARTER, ZoneId.of("Europe/Kyiv"))).isEqualTo(1719781200000L); // Monday, July 1, 2024 0:00:00 GMT+03:00 DST + assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.QUARTER, ZoneId.of("America/New_York"))).isEqualTo(1711944000000L); // Monday, April 1, 2024 7:00:00 GMT+03:00 DST + } + +} From ee08f33d98faa42045366229af338ac03684cd4d Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 11 Jan 2024 12:51:52 +0200 Subject: [PATCH 3/8] TimeZone name links --- .../server/common/data/kv/AggregationParams.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/AggregationParams.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/AggregationParams.java index 9d7470b676..6c85cfb3c0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/AggregationParams.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/AggregationParams.java @@ -19,18 +19,18 @@ import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; import org.thingsboard.server.common.data.StringUtils; import java.time.DateTimeException; import java.time.ZoneId; -import java.time.zone.ZoneRulesException; +import java.util.Map; import java.util.concurrent.TimeUnit; @AllArgsConstructor @EqualsAndHashCode @Slf4j public class AggregationParams { + private static final Map TZ_LINKS = Map.of("EST", "America/New_York", "GMT+0", "GMT", "GMT-0", "GMT", "HST", "US/Hawaii", "MST", "America/Phoenix", "ROC", "Asia/Taipei"); @Getter private final Aggregation aggregation; @Getter @@ -83,7 +83,7 @@ public class AggregationParams { return ZoneId.systemDefault(); } try { - return ZoneId.of(tzIdStr); + return ZoneId.of(tzIdStr, TZ_LINKS); } catch (DateTimeException e) { log.warn("[{}] Failed to convert the time zone. Fallback to default.", tzIdStr); return ZoneId.systemDefault(); From 56792bba97b62bfc9df5304fa57fe826c3a9cd44 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 12 Jan 2024 15:04:05 +0200 Subject: [PATCH 4/8] UI: Introduce new aggregation intervals. Improve data intervals processing and aggregation. --- .../widget_types/bar_chart_with_labels.json | 2 +- .../server/ThingsboardServerApplication.java | 6 +- ui-ngx/package.json | 1 + ui-ngx/src/app/core/api/data-aggregator.ts | 161 +++++++++------- .../app/core/api/entity-data-subscription.ts | 96 ++++++---- .../src/app/core/api/widget-subscription.ts | 9 +- ui-ngx/src/app/core/services/time.service.ts | 67 ++++--- .../lib/cards/aggregated-value-card.models.ts | 4 +- .../bar-chart-with-labels-widget.component.ts | 51 ++--- .../widget/lib/chart/echarts-widget.models.ts | 54 ++++-- .../lib/chart/range-chart-widget.component.ts | 3 + .../home/components/widget/lib/flot-widget.ts | 8 +- .../liquid-level-widget.component.ts | 5 +- .../time/timeinterval.component.html | 2 +- .../components/time/timeinterval.component.ts | 70 +++---- .../time/timewindow-panel.component.html | 2 + .../models/telemetry/telemetry.models.ts | 18 +- .../src/app/shared/models/time/time.models.ts | 178 +++++++++++++++--- .../shared/models/widget-settings.models.ts | 16 +- ui-ngx/src/app/shared/models/widget.models.ts | 8 +- .../assets/locale/locale.constant-en_US.json | 6 + ui-ngx/yarn.lock | 5 + 22 files changed, 522 insertions(+), 250 deletions(-) diff --git a/application/src/main/data/json/system/widget_types/bar_chart_with_labels.json b/application/src/main/data/json/system/widget_types/bar_chart_with_labels.json index 06ee881c0a..75814a547a 100644 --- a/application/src/main/data/json/system/widget_types/bar_chart_with_labels.json +++ b/application/src/main/data/json/system/widget_types/bar_chart_with_labels.json @@ -20,7 +20,7 @@ "latestDataKeySettingsDirective": "", "hasBasicMode": true, "basicModeDirective": "tb-bar-chart-with-labels-basic-config", - "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Humidity\",\"color\":\"rgb(125, 142, 255)\",\"settings\":{},\"_hash\":0.8587686344902596,\"funcBody\":\"var value = prevValue + Math.random() * 40 - 20;\\nif (value < 50) {\\n\\tvalue = 50;\\n} else if (value > 80) {\\n\\tvalue = 80;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Soil moisture\",\"color\":\"rgb(249, 111, 255)\",\"settings\":{},\"_hash\":0.9111685461089025,\"funcBody\":\"var value = prevValue + Math.random() * 40 - 20;\\nif (value < 30) {\\n\\tvalue = 30;\\n} else if (value > 90) {\\n\\tvalue = 90;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Leaf wetness\",\"color\":\"rgb(255, 163, 137)\",\"settings\":{},\"_hash\":0.8487533373085416,\"funcBody\":\"var value = prevValue + Math.random() * 40 - 20;\\nif (value < 40) {\\n\\tvalue = 40;\\n} else if (value > 70) {\\n\\tvalue = 70;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Cloud cover\",\"color\":\"#FFED53\",\"settings\":{},\"_hash\":0.7690144858984289,\"funcBody\":\"var value = prevValue + Math.random() * 40 - 20;\\nif (value < 20) {\\n\\tvalue = 20;\\n} else if (value > 60) {\\n\\tvalue = 60;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null}],\"alarmFilterConfig\":{\"statusList\":[\"ACTIVE\"]}}],\"timewindow\":{\"hideInterval\":false,\"hideLastInterval\":false,\"hideQuickInterval\":false,\"hideAggregation\":false,\"hideAggInterval\":false,\"hideTimezone\":false,\"selectedTab\":1,\"history\":{\"historyType\":2,\"timewindowMs\":60000,\"interval\":2592000000,\"fixedTimewindow\":{\"startTimeMs\":1704293713163,\"endTimeMs\":1704380113163},\"quickInterval\":\"CURRENT_HALF_YEAR\"},\"aggregation\":{\"type\":\"AVG\",\"limit\":25000},\"timezone\":null},\"showTitle\":true,\"backgroundColor\":\"rgba(0, 0, 0, 0)\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"0px\",\"settings\":{\"showBarLabel\":true,\"barLabelFont\":{\"family\":\"Roboto\",\"size\":12,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"400\",\"lineHeight\":\"12px\"},\"barLabelColor\":\"rgba(0, 0, 0, 0.54)\",\"showBarValue\":true,\"barValueFont\":{\"family\":\"Roboto\",\"size\":12,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"700\",\"lineHeight\":\"12px\"},\"barValueColor\":\"rgba(0, 0, 0, 0.76)\",\"showLegend\":true,\"legendPosition\":\"top\",\"legendLabelFont\":{\"family\":\"Roboto\",\"size\":12,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"400\",\"lineHeight\":\"16px\"},\"legendLabelColor\":\"rgba(0, 0, 0, 0.76)\",\"showTooltip\":true,\"tooltipValueFont\":{\"family\":\"Roboto\",\"size\":12,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"500\",\"lineHeight\":\"16px\"},\"tooltipValueColor\":\"rgba(0, 0, 0, 0.76)\",\"tooltipShowDate\":true,\"tooltipDateFormat\":{\"format\":\"MMMM y\",\"lastUpdateAgo\":false,\"custom\":true},\"tooltipDateFont\":{\"family\":\"Roboto\",\"size\":11,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"400\",\"lineHeight\":\"16px\"},\"tooltipDateColor\":\"rgba(0, 0, 0, 0.76)\",\"tooltipBackgroundColor\":\"rgba(255, 255, 255, 0.76)\",\"tooltipBackgroundBlur\":4,\"background\":{\"type\":\"color\",\"color\":\"#fff\",\"overlay\":{\"enabled\":false,\"color\":\"rgba(255,255,255,0.72)\",\"blur\":3}}},\"title\":\"Bar chart with labels\",\"dropShadow\":true,\"enableFullscreen\":true,\"titleStyle\":null,\"mobileHeight\":null,\"configMode\":\"basic\",\"actions\":{},\"showTitleIcon\":false,\"titleIcon\":\"public\",\"iconColor\":\"#1F6BDD\",\"useDashboardTimewindow\":false,\"displayTimewindow\":true,\"titleFont\":{\"size\":16,\"sizeUnit\":\"px\",\"family\":\"Roboto\",\"weight\":\"500\",\"style\":\"normal\",\"lineHeight\":\"24px\"},\"titleColor\":\"rgba(0, 0, 0, 0.87)\",\"titleTooltip\":\"\",\"widgetStyle\":{},\"widgetCss\":\"\",\"pageSize\":1024,\"units\":\"%\",\"decimals\":0,\"noDataDisplayMessage\":\"\",\"timewindowStyle\":{\"showIcon\":false,\"iconSize\":\"24px\",\"icon\":null,\"iconPosition\":\"left\",\"font\":{\"size\":12,\"sizeUnit\":\"px\",\"family\":\"Roboto\",\"weight\":\"400\",\"style\":\"normal\",\"lineHeight\":\"16px\"},\"color\":\"rgba(0, 0, 0, 0.38)\",\"displayTypePrefix\":true},\"margin\":\"0px\",\"borderRadius\":\"0px\",\"iconSize\":\"24px\"}" + "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Humidity\",\"color\":\"rgb(125, 142, 255)\",\"settings\":{},\"_hash\":0.8587686344902596,\"funcBody\":\"var value = prevValue + Math.random() * 40 - 20;\\nif (value < 50) {\\n\\tvalue = 50;\\n} else if (value > 80) {\\n\\tvalue = 80;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Soil moisture\",\"color\":\"rgb(249, 111, 255)\",\"settings\":{},\"_hash\":0.9111685461089025,\"funcBody\":\"var value = prevValue + Math.random() * 40 - 20;\\nif (value < 30) {\\n\\tvalue = 30;\\n} else if (value > 90) {\\n\\tvalue = 90;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Leaf wetness\",\"color\":\"rgb(255, 163, 137)\",\"settings\":{},\"_hash\":0.8487533373085416,\"funcBody\":\"var value = prevValue + Math.random() * 40 - 20;\\nif (value < 40) {\\n\\tvalue = 40;\\n} else if (value > 70) {\\n\\tvalue = 70;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Cloud cover\",\"color\":\"#FFED53\",\"settings\":{},\"_hash\":0.7690144858984289,\"funcBody\":\"var value = prevValue + Math.random() * 40 - 20;\\nif (value < 20) {\\n\\tvalue = 20;\\n} else if (value > 60) {\\n\\tvalue = 60;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null}],\"alarmFilterConfig\":{\"statusList\":[\"ACTIVE\"]}}],\"timewindow\":{\"hideInterval\":false,\"hideLastInterval\":false,\"hideQuickInterval\":false,\"hideAggregation\":false,\"hideAggInterval\":false,\"hideTimezone\":false,\"selectedTab\":1,\"history\":{\"historyType\":2,\"timewindowMs\":60000,\"interval\":\"MONTH\",\"fixedTimewindow\":{\"startTimeMs\":1704293713163,\"endTimeMs\":1704380113163},\"quickInterval\":\"CURRENT_HALF_YEAR\"},\"aggregation\":{\"type\":\"AVG\",\"limit\":25000},\"timezone\":null},\"showTitle\":true,\"backgroundColor\":\"rgba(0, 0, 0, 0)\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"0px\",\"settings\":{\"showBarLabel\":true,\"barLabelFont\":{\"family\":\"Roboto\",\"size\":12,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"400\",\"lineHeight\":\"12px\"},\"barLabelColor\":\"rgba(0, 0, 0, 0.54)\",\"showBarValue\":true,\"barValueFont\":{\"family\":\"Roboto\",\"size\":12,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"700\",\"lineHeight\":\"12px\"},\"barValueColor\":\"rgba(0, 0, 0, 0.76)\",\"showLegend\":true,\"legendPosition\":\"top\",\"legendLabelFont\":{\"family\":\"Roboto\",\"size\":12,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"400\",\"lineHeight\":\"16px\"},\"legendLabelColor\":\"rgba(0, 0, 0, 0.76)\",\"showTooltip\":true,\"tooltipValueFont\":{\"family\":\"Roboto\",\"size\":12,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"500\",\"lineHeight\":\"16px\"},\"tooltipValueColor\":\"rgba(0, 0, 0, 0.76)\",\"tooltipShowDate\":true,\"tooltipDateFormat\":{\"format\":\"MMMM y\",\"lastUpdateAgo\":false,\"custom\":true},\"tooltipDateFont\":{\"family\":\"Roboto\",\"size\":11,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"400\",\"lineHeight\":\"16px\"},\"tooltipDateColor\":\"rgba(0, 0, 0, 0.76)\",\"tooltipBackgroundColor\":\"rgba(255, 255, 255, 0.76)\",\"tooltipBackgroundBlur\":4,\"background\":{\"type\":\"color\",\"color\":\"#fff\",\"overlay\":{\"enabled\":false,\"color\":\"rgba(255,255,255,0.72)\",\"blur\":3}}},\"title\":\"Bar chart with labels\",\"dropShadow\":true,\"enableFullscreen\":true,\"titleStyle\":null,\"mobileHeight\":null,\"configMode\":\"basic\",\"actions\":{},\"showTitleIcon\":false,\"titleIcon\":\"public\",\"iconColor\":\"#1F6BDD\",\"useDashboardTimewindow\":false,\"displayTimewindow\":true,\"titleFont\":{\"size\":16,\"sizeUnit\":\"px\",\"family\":\"Roboto\",\"weight\":\"500\",\"style\":\"normal\",\"lineHeight\":\"24px\"},\"titleColor\":\"rgba(0, 0, 0, 0.87)\",\"titleTooltip\":\"\",\"widgetStyle\":{},\"widgetCss\":\"\",\"pageSize\":1024,\"units\":\"%\",\"decimals\":0,\"noDataDisplayMessage\":\"\",\"timewindowStyle\":{\"showIcon\":false,\"iconSize\":\"24px\",\"icon\":null,\"iconPosition\":\"left\",\"font\":{\"size\":12,\"sizeUnit\":\"px\",\"family\":\"Roboto\",\"weight\":\"400\",\"style\":\"normal\",\"lineHeight\":\"16px\"},\"color\":\"rgba(0, 0, 0, 0.38)\",\"displayTypePrefix\":true},\"margin\":\"0px\",\"borderRadius\":\"0px\",\"iconSize\":\"24px\"}" }, "tags": [ "bar chart", diff --git a/application/src/main/java/org/thingsboard/server/ThingsboardServerApplication.java b/application/src/main/java/org/thingsboard/server/ThingsboardServerApplication.java index 77442e53a8..60f69cd8a7 100644 --- a/application/src/main/java/org/thingsboard/server/ThingsboardServerApplication.java +++ b/application/src/main/java/org/thingsboard/server/ThingsboardServerApplication.java @@ -21,7 +21,6 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; -import java.time.ZoneId; import java.util.Arrays; @SpringBootConfiguration @@ -34,10 +33,7 @@ public class ThingsboardServerApplication { private static final String DEFAULT_SPRING_CONFIG_PARAM = SPRING_CONFIG_NAME_KEY + "=" + "thingsboard"; public static void main(String[] args) { - - ZoneId.getAvailableZoneIds().stream().sorted().forEach(System.out::println); - -// SpringApplication.run(ThingsboardServerApplication.class, updateArguments(args)); + SpringApplication.run(ThingsboardServerApplication.class, updateArguments(args)); } private static String[] updateArguments(String[] args) { diff --git a/ui-ngx/package.json b/ui-ngx/package.json index f32b68590a..4bbe817fcf 100644 --- a/ui-ngx/package.json +++ b/ui-ngx/package.json @@ -97,6 +97,7 @@ "rxjs": "~7.8.0", "schema-inspector": "^2.0.2", "screenfull": "^6.0.2", + "sorted-btree": "^1.8.1", "split.js": "^1.6.5", "systemjs": "6.14.1", "tinycolor2": "^1.6.0", diff --git a/ui-ngx/src/app/core/api/data-aggregator.ts b/ui-ngx/src/app/core/api/data-aggregator.ts index a09b1e984a..a16c4baa5b 100644 --- a/ui-ngx/src/app/core/api/data-aggregator.ts +++ b/ui-ngx/src/app/core/api/data-aggregator.ts @@ -19,36 +19,40 @@ import { IndexedSubscriptionData, } from '@app/shared/models/telemetry/telemetry.models'; import { - AggregationType, + AggregationType, calculateAggInterval, calculateIntervalComparisonEndTime, calculateIntervalEndTime, calculateIntervalStartEndTime, getCurrentTime, - getTime, + getTime, IntervalMath, SubscriptionTimewindow } from '@shared/models/time/time.models'; import { UtilsService } from '@core/services/utils.service'; import { deepClone, isDefinedAndNotNull, isNumber, isNumeric } from '@core/utils'; import Timeout = NodeJS.Timeout; +import { DataEntry, DataSet, IndexedData } from '@shared/models/widget.models'; +import BTree from 'sorted-btree'; -export declare type onAggregatedData = (data: IndexedSubscriptionData, detectChanges: boolean) => void; +export declare type onAggregatedData = (data: IndexedData, detectChanges: boolean) => void; interface AggData { count: number; sum: number; aggValue: any; + ts: number; + interval: [number, number]; } class AggDataMap { - rangeChanged = false; - private minTs = Number.MAX_SAFE_INTEGER; - private map = new Map(); + private map = new BTree(); + private reusePair: [number, AggData] = [undefined, undefined]; + + constructor( + private subsTw: SubscriptionTimewindow, + private endTs: number + ){}; set(ts: number, data: AggData) { - if (ts < this.minTs) { - this.rangeChanged = true; - this.minTs = ts; - } this.map.set(ts, data); } @@ -60,7 +64,38 @@ class AggDataMap { this.map.delete(ts); } - forEach(callback: (value: AggData, key: number, map: Map) => void, thisArg?: any) { + findDataForTs(ts: number): AggData | undefined { + if (ts >= this.endTs) { + this.updateLastInterval(ts + 1); + } + const pair = this.map.getPairOrNextLower(ts, this.reusePair); + if (pair) { + const data = pair[1]; + const interval = data.interval; + if (ts < interval[1]) { + return data; + } + } + } + + calculateAggInterval(timestamp: number): [number, number] { + return calculateAggInterval(this.subsTw, this.endTs, timestamp); + } + + updateLastInterval(endTs: number) { + if (endTs > this.endTs) { + this.endTs = endTs; + const lastTs = this.map.maxKey(); + if (lastTs) { + const data = this.map.get(lastTs); + const interval = calculateAggInterval(this.subsTw, endTs, data.ts); + data.interval = interval; + data.ts = interval[0] + Math.floor((interval[1] - interval[0]) / 2); + } + } + } + + forEach(callback: (value: AggData, key: number, map: BTree) => void, thisArg?: any) { this.map.forEach(callback, thisArg); } @@ -71,24 +106,6 @@ class AggDataMap { class AggregationMap { aggMap: {[id: number]: AggDataMap} = {}; - - detectRangeChanged(): boolean { - let changed = false; - for (const id of Object.keys(this.aggMap)) { - const aggDataMap = this.aggMap[id]; - if (aggDataMap.rangeChanged) { - changed = true; - aggDataMap.rangeChanged = false; - } - } - return changed; - } - - clearRangeChangedFlags() { - for (const id of Object.keys(this.aggMap)) { - this.aggMap[id].rangeChanged = false; - } - } } declare type AggFunction = (aggData: AggData, value?: any) => void; @@ -156,9 +173,9 @@ export class DataAggregator { } } - private dataBuffer: IndexedSubscriptionData = []; - private data: IndexedSubscriptionData; - private readonly lastPrevKvPairData: {[id: number]: [number, any]}; + private dataBuffer: IndexedData = []; + private data: IndexedData; + private readonly lastPrevKvPairData: {[id: number]: DataEntry}; private aggregationMap: AggregationMap; @@ -166,7 +183,7 @@ export class DataAggregator { private resetPending = false; private updatedData = false; - private aggregationTimeout = this.isLatestDataAgg ? 1000 : Math.max(this.subsTw.aggregation.interval, 1000); + private aggregationTimeout = this.isLatestDataAgg ? 1000 : Math.max(IntervalMath.numberValue(this.subsTw.aggregation.interval), 1000); private intervalTimeoutHandle: Timeout; private intervalScheduledTime: number; @@ -216,7 +233,7 @@ export class DataAggregator { this.intervalScheduledTime = this.utils.currentPerfTime(); this.calculateStartEndTs(); this.elapsed = 0; - this.aggregationTimeout = this.isLatestDataAgg ? 1000 : Math.max(this.subsTw.aggregation.interval, 1000); + this.aggregationTimeout = this.isLatestDataAgg ? 1000 : Math.max(IntervalMath.numberValue(this.subsTw.aggregation.interval), 1000); this.resetPending = true; this.updatedData = false; this.intervalTimeoutHandle = setTimeout(this.onInterval.bind(this), Math.min(this.aggregationTimeout, MAX_INTERVAL_TIMEOUT)); @@ -252,7 +269,6 @@ export class DataAggregator { if (updateIntervalScheduledTime) { this.intervalScheduledTime = this.utils.currentPerfTime(); } - this.aggregationMap.clearRangeChangedFlags(); this.onInterval(history, detectChanges); } else { this.updateAggregatedData(data); @@ -260,9 +276,7 @@ export class DataAggregator { this.intervalScheduledTime = this.utils.currentPerfTime(); this.onInterval(history, detectChanges); } else { - if (this.aggregationMap.detectRangeChanged()) { - this.onInterval(false, detectChanges, true); - } + this.onInterval(false, detectChanges, true); } } } @@ -283,7 +297,7 @@ export class DataAggregator { } } - private onInterval(history?: boolean, detectChanges?: boolean, rangeChanged?: boolean) { + private onInterval(history?: boolean, detectChanges?: boolean, dataChanged?: boolean) { const now = this.utils.currentPerfTime(); this.elapsed += now - this.intervalScheduledTime; this.intervalScheduledTime = now; @@ -291,10 +305,10 @@ export class DataAggregator { clearTimeout(this.intervalTimeoutHandle); this.intervalTimeoutHandle = null; } - const intervalTimeout = rangeChanged ? this.aggregationTimeout - this.elapsed : this.aggregationTimeout; + const intervalTimeout = dataChanged ? this.aggregationTimeout - this.elapsed : this.aggregationTimeout; if (!history) { const delta = Math.floor(this.elapsed / this.aggregationTimeout); - if (delta || !this.data || rangeChanged) { + if (delta || !this.data || dataChanged) { const tickTs = delta * this.aggregationTimeout; if (this.subsTw.quickInterval) { const startEndTime = calculateIntervalStartEndTime(this.subsTw.quickInterval, this.subsTw.timezone); @@ -304,6 +318,7 @@ export class DataAggregator { this.startTs += tickTs; this.endTs += tickTs; } + this.updateLastInterval(); this.data = this.updateData(); this.elapsed = this.elapsed - delta * this.aggregationTimeout; } @@ -319,7 +334,7 @@ export class DataAggregator { } } - private updateData(): IndexedSubscriptionData { + private updateData(): IndexedData { this.dataBuffer = []; this.tsKeys.forEach((key) => { if (!this.dataBuffer[key.id]) { @@ -332,19 +347,21 @@ export class DataAggregator { const aggKey = this.aggKeyById(id); const noAggregation = aggKey.agg === AggregationType.NONE; let keyData = this.dataBuffer[id]; - aggKeyData.forEach((aggData, aggTimestamp) => { - if (aggTimestamp < this.startTs) { + const deletedKeys: number[] = []; + aggKeyData.forEach((aggData, aggStartTs) => { + if (aggStartTs < this.startTs) { if (this.subsTw.aggregation.stateData && - (!this.lastPrevKvPairData[id] || this.lastPrevKvPairData[id][0] < aggTimestamp)) { - this.lastPrevKvPairData[id] = [aggTimestamp, aggData.aggValue]; + (!this.lastPrevKvPairData[id] || this.lastPrevKvPairData[id][0] < aggData.ts)) { + this.lastPrevKvPairData[id] = [aggData.ts, aggData.aggValue, aggData.interval]; } - aggKeyData.delete(aggTimestamp); + deletedKeys.push(aggStartTs); this.updatedData = true; - } else if (aggTimestamp < this.endTs || noAggregation) { - const kvPair: [number, any] = [aggTimestamp, aggData.aggValue]; + } else if (aggData.ts < this.endTs || noAggregation) { + const kvPair: DataEntry = [aggData.ts, aggData.aggValue, aggData.interval]; keyData.push(kvPair); } }); + deletedKeys.forEach(ts => aggKeyData.delete(ts)); keyData.sort((set1, set2) => set1[0] - set2[0]); if (this.subsTw.aggregation.stateData) { this.updateStateBounds(keyData, deepClone(this.lastPrevKvPairData[id])); @@ -357,11 +374,11 @@ export class DataAggregator { return this.dataBuffer; } - private updateStateBounds(keyData: [number, any, number?][], lastPrevKvPair: [number, any]) { + private updateStateBounds(keyData: DataSet, lastPrevKvPair: DataEntry) { if (lastPrevKvPair) { lastPrevKvPair[0] = this.startTs; } - let firstKvPair; + let firstKvPair: DataEntry; if (!keyData.length) { if (lastPrevKvPair) { firstKvPair = lastPrevKvPair; @@ -395,20 +412,26 @@ export class DataAggregator { const noAggregation = aggType === AggregationType.NONE; let aggKeyData = aggregationMap.aggMap[id]; if (!aggKeyData) { - aggKeyData = new AggDataMap(); + aggKeyData = new AggDataMap(this.subsTw, this.endTs); aggregationMap.aggMap[id] = aggKeyData; } const keyData = data[id]; keyData.forEach((kvPair) => { const timestamp = kvPair[0]; const value = DataAggregator.convertValue(kvPair[1], noAggregation); - const tsKey = timestamp; - const aggData = { + let interval: [number, number] = [timestamp, timestamp]; + if (!noAggregation) { + interval = aggKeyData.calculateAggInterval(timestamp); + } + const ts = interval[0] + Math.floor((interval[1] - interval[0]) / 2); + const aggData: AggData = { count: isCount ? value : isDefinedAndNotNull(kvPair[2]) ? kvPair[2] : 1, sum: value, - aggValue: value + aggValue: value, + ts, + interval }; - aggKeyData.set(tsKey, aggData); + aggKeyData.set(interval[0], aggData); }); } return aggregationMap; @@ -423,24 +446,28 @@ export class DataAggregator { const noAggregation = aggType === AggregationType.NONE; let aggKeyData = this.aggregationMap.aggMap[id]; if (!aggKeyData) { - aggKeyData = new AggDataMap(); + aggKeyData = new AggDataMap(this.subsTw, this.endTs); this.aggregationMap.aggMap[id] = aggKeyData; } const keyData = data[id]; keyData.forEach((kvPair) => { const timestamp = kvPair[0]; const value = DataAggregator.convertValue(kvPair[1], noAggregation); - const aggTimestamp = noAggregation ? timestamp : (this.startTs + - Math.floor((timestamp - this.startTs) / this.subsTw.aggregation.interval) * - this.subsTw.aggregation.interval + this.subsTw.aggregation.interval / 2); - let aggData = aggKeyData.get(aggTimestamp); + let aggData = aggKeyData.findDataForTs(timestamp); if (!aggData) { + let interval: [number, number] = [timestamp, timestamp]; + if (!noAggregation) { + interval = aggKeyData.calculateAggInterval(timestamp); + } + const ts = interval[0] + Math.floor((interval[1] - interval[0]) / 2); aggData = { count: isDefinedAndNotNull(kvPair[2]) ? kvPair[2] : 1, sum: value, - aggValue: isCount ? 1 : value + aggValue: isCount ? 1 : value, + ts, + interval }; - aggKeyData.set(aggTimestamp, aggData); + aggKeyData.set(interval[0], aggData); } else { DataAggregator.getAggFunction(aggType)(aggData, value); } @@ -448,6 +475,14 @@ export class DataAggregator { } } + private updateLastInterval() { + for (const idStr of Object.keys(this.aggregationMap.aggMap)) { + const id = Number(idStr); + const aggKeyData = this.aggregationMap.aggMap[id]; + aggKeyData.updateLastInterval(this.endTs); + } + } + private aggKeyById(id: number): AggKey { return this.tsKeys.find(key => key.id === id); } diff --git a/ui-ngx/src/app/core/api/entity-data-subscription.ts b/ui-ngx/src/app/core/api/entity-data-subscription.ts index af3354c07b..a1c88b3330 100644 --- a/ui-ngx/src/app/core/api/entity-data-subscription.ts +++ b/ui-ngx/src/app/core/api/entity-data-subscription.ts @@ -14,12 +14,19 @@ /// limitations under the License. /// -import { ComparisonResultType, DataSet, DataSetHolder, DatasourceType, widgetType } from '@shared/models/widget.models'; +import { + ComparisonResultType, + DataEntry, + DataSet, + DataSetHolder, + DatasourceType, IndexedData, + widgetType +} from '@shared/models/widget.models'; import { AggregationType, ComparisonDuration, createTimewindowForComparison, - getCurrentTime, + getCurrentTime, IntervalMath, SubscriptionTimewindow } from '@shared/models/time/time.models'; import { @@ -42,15 +49,16 @@ import { EntityCountCmd, EntityDataCmd, IndexedSubscriptionData, + IntervalType, NOT_SUPPORTED, - SubscriptionData, + SubscriptionData, SubscriptionDataEntry, TelemetrySubscriber } from '@shared/models/telemetry/telemetry.models'; import { UtilsService } from '@core/services/utils.service'; import { EntityDataListener, EntityDataLoadResult } from '@core/api/entity-data.service'; import { deepClone, isDefined, isDefinedAndNotNull, isNumeric, isObject, objectHashCode } from '@core/utils'; import { PageData } from '@shared/models/page/page-data'; -import { DataAggregator } from '@core/api/data-aggregator'; +import { DataAggregator, onAggregatedData } from '@core/api/data-aggregator'; import { NULL_UUID } from '@shared/models/id/has-uuid'; import { EntityType } from '@shared/models/entity-type.models'; import { Observable, of, ReplaySubject, Subject } from 'rxjs'; @@ -154,7 +162,7 @@ export class EntityDataSubscription { return val; } - private static calculateComparisonValue(key: SubscriptionDataKey, comparisonTsValue: ComparisonTsValue): [number, any, number?][] { + private static calculateComparisonValue(key: SubscriptionDataKey, comparisonTsValue: ComparisonTsValue): DataSet { let timestamp: number; let value: any; switch (key.comparisonResultType) { @@ -385,7 +393,13 @@ export class EntityDataSubscription { if (this.entityDataSubscriptionOptions.type === widgetType.timeseries && this.tsFields.length) { targetCommand.tsCmd.startTs = this.subsTw.startTs; targetCommand.tsCmd.timeWindow = this.subsTw.aggregation.timeWindow; - targetCommand.tsCmd.interval = this.subsTw.aggregation.interval; + if (typeof this.subsTw.aggregation.interval === 'number') { + targetCommand.tsCmd.interval = this.subsTw.aggregation.interval; + targetCommand.tsCmd.intervalType = IntervalType.MILLISECONDS; + } else { + targetCommand.tsCmd.intervalType = this.subsTw.aggregation.interval; + } + targetCommand.tsCmd.timeZoneId = this.subsTw.timezone; targetCommand.tsCmd.limit = this.subsTw.aggregation.limit; targetCommand.tsCmd.agg = this.subsTw.aggregation.type; targetCommand.tsCmd.fetchLatestPreviousPoint = this.subsTw.aggregation.stateData; @@ -659,21 +673,35 @@ export class EntityDataSubscription { keys: this.tsFields.map(key => key.key), startTs: this.subsTw.fixedWindow.startTimeMs, endTs: this.subsTw.fixedWindow.endTimeMs, - interval: this.subsTw.aggregation.interval, + interval: 0, + intervalType: IntervalType.MILLISECONDS, limit: this.subsTw.aggregation.limit, + timeZoneId: this.subsTw.timezone, agg: this.subsTw.aggregation.type, fetchLatestPreviousPoint: this.subsTw.aggregation.stateData }; + if (typeof this.subsTw.aggregation.interval === 'number') { + cmd.historyCmd.interval = this.subsTw.aggregation.interval; + } else { + cmd.historyCmd.intervalType = this.subsTw.aggregation.interval; + } } else { cmd.tsCmd = { keys: this.tsFields.map(key => key.key), startTs: this.subsTw.startTs, timeWindow: this.subsTw.aggregation.timeWindow, - interval: this.subsTw.aggregation.interval, + interval: 0, + intervalType: IntervalType.MILLISECONDS, limit: this.subsTw.aggregation.limit, + timeZoneId: this.subsTw.timezone, agg: this.subsTw.aggregation.type, fetchLatestPreviousPoint: this.subsTw.aggregation.stateData }; + if (typeof this.subsTw.aggregation.interval === 'number') { + cmd.tsCmd.interval = this.subsTw.aggregation.interval; + } else { + cmd.tsCmd.intervalType = this.subsTw.aggregation.interval; + } } } latestValuesKeys = this.latestValues; @@ -717,7 +745,8 @@ export class EntityDataSubscription { this.frequency = 1000; this.latestFrequency = 1000; if (this.entityDataSubscriptionOptions.type === widgetType.timeseries) { - this.frequency = Math.min(this.entityDataSubscriptionOptions.subscriptionTimewindow.aggregation.interval, 5000); + this.frequency = + Math.min(IntervalMath.numberValue(this.entityDataSubscriptionOptions.subscriptionTimewindow.aggregation.interval), 5000); } this.tickScheduledTime = this.utils.currentPerfTime(); this.generateData(true); @@ -785,9 +814,9 @@ export class EntityDataSubscription { } private reportNotSupported(keys: AggKey[], isUpdate: boolean) { - const indexedData: IndexedSubscriptionData = []; + const indexedData: IndexedData = []; for (const key of keys) { - indexedData[key.id] = [[0, NOT_SUPPORTED]]; + indexedData[key.id] = [[0, NOT_SUPPORTED, [0,0]]]; } for (let dataIndex = 0; dataIndex < this.pageData.data.length; dataIndex++) { this.onIndexedData(indexedData, dataIndex, true, @@ -942,7 +971,7 @@ export class EntityDataSubscription { } if (Object.keys(aggData).length > 0 && this.tsLatestDataAggregators && this.tsLatestDataAggregators[dataIndex]) { const dataAggregator = this.tsLatestDataAggregators[dataIndex]; - let prevDataCb; + let prevDataCb: onAggregatedData; if (!isUpdate) { prevDataCb = dataAggregator.updateOnDataCb((data, detectChanges) => { this.onIndexedData(data, dataIndex, detectChanges, @@ -996,7 +1025,7 @@ export class EntityDataSubscription { for (const dataKey of dataKeys) { indexedData[dataKey.index] = subscriptionData[dataKey.name]; } - let prevDataCb; + let prevDataCb: onAggregatedData; if (!isUpdate) { prevDataCb = dataAggregator.updateOnDataCb((data, detectChanges) => { this.onIndexedData(data, dataIndex, detectChanges, false, dataUpdatedCb); @@ -1016,12 +1045,12 @@ export class EntityDataSubscription { isTsLatest: boolean, dataUpdatedCb: DataUpdatedCb) { for (const key of Object.keys(sourceData)) { const keyData = sourceData[key]; - this.onKeyData(keyData, key, 0, type, + this.onKeyData(keyData.map(entry => [entry[0], entry[1], [entry[0], entry[0]]]), key, 0, type, dataIndex, detectChanges, isTsLatest, false, dataUpdatedCb); } } - private onIndexedData(sourceData: IndexedSubscriptionData, dataIndex: number, detectChanges: boolean, + private onIndexedData(sourceData: IndexedData, dataIndex: number, detectChanges: boolean, isTsLatest: boolean, dataUpdatedCb: DataUpdatedCb) { for (const indexStr of Object.keys(sourceData)) { const id = Number(indexStr); @@ -1037,7 +1066,7 @@ export class EntityDataSubscription { } } - private onKeyData(keyData: [number, any, number?][], keyName: string, id: number, type: DataKeyType, + private onKeyData(keyData: DataSet, keyName: string, id: number, type: DataKeyType, dataIndex: number, detectChanges: boolean, isTsLatest: boolean, isAggLatest: boolean, dataUpdatedCb: DataUpdatedCb) { const keyIdSuffix = isAggLatest ? `_${id}` : ''; @@ -1048,8 +1077,8 @@ export class EntityDataSubscription { if (this.datasourceData[dataIndex][datasourceKey].data) { const dataKey = dataKeyList[keyIndex]; const data: DataSet = []; - let prevSeries: [number, any]; - let prevOrigSeries: [number, any]; + let prevSeries: DataEntry; + let prevOrigSeries: DataEntry; let datasourceKeyData: DataSet; let datasourceOrigKeyData: DataSet; let update = false; @@ -1064,36 +1093,36 @@ export class EntityDataSubscription { prevSeries = datasourceKeyData[datasourceKeyData.length - 1]; prevOrigSeries = datasourceOrigKeyData[datasourceOrigKeyData.length - 1]; } else { - prevSeries = [0, 0]; - prevOrigSeries = [0, 0]; + prevSeries = [0, 0, [0, 0]]; + prevOrigSeries = [0, 0, [0, 0]]; } this.datasourceOrigData[dataIndex][datasourceKey].data = []; if (this.entityDataSubscriptionOptions.type === widgetType.timeseries && !isTsLatest) { keyData.forEach((keySeries) => { let series = keySeries; const time = series[0]; - this.datasourceOrigData[dataIndex][datasourceKey].data.push([series[0], series[1]]); + this.datasourceOrigData[dataIndex][datasourceKey].data.push([series[0], series[1], series[2]]); let value = EntityDataSubscription.convertValue(series[1]); if (dataKey.postFunc) { value = dataKey.postFunc(time, value, prevSeries[1], prevOrigSeries[0], prevOrigSeries[1]); } - prevOrigSeries = [series[0], series[1]]; - series = [series[0], value]; - data.push([series[0], series[1]]); - prevSeries = [series[0], series[1]]; + prevOrigSeries = [series[0], series[1], series[2]]; + series = [series[0], value, series[2]]; + data.push([series[0], series[1], series[2]]); + prevSeries = [series[0], series[1], series[2]]; }); update = true; } else if (this.entityDataSubscriptionOptions.type === widgetType.latest || isTsLatest) { if (keyData.length > 0) { let series = keyData[0]; const time = series[0]; - this.datasourceOrigData[dataIndex][datasourceKey].data.push([series[0], series[1]]); + this.datasourceOrigData[dataIndex][datasourceKey].data.push([series[0], series[1], series[2]]); let value = EntityDataSubscription.convertValue(series[1]); if (dataKey.postFunc) { value = dataKey.postFunc(time, value, prevSeries[1], prevOrigSeries[0], prevOrigSeries[1]); } - series = [time, value]; - data.push([series[0], series[1]]); + series = [time, value, series[2]]; + data.push([series[0], series[1], series[2]]); } update = true; } @@ -1155,13 +1184,14 @@ export class EntityDataSubscription { return result; } - private generateSeries(dataKey: SubscriptionDataKey, startTime: number, endTime: number): [number, any][] { - const data: [number, any][] = []; - let prevSeries: [number, any]; + private generateSeries(dataKey: SubscriptionDataKey, startTime: number, endTime: number): SubscriptionDataEntry[] { + const data: SubscriptionDataEntry[] = []; + let prevSeries: SubscriptionDataEntry; const datasourceDataKey = `${dataKey.key}_${dataKey.listIndex}`; const datasourceKeyData = this.datasourceData[0][datasourceDataKey].data; if (datasourceKeyData.length > 0) { - prevSeries = datasourceKeyData[datasourceKeyData.length - 1]; + const prevDataEntry = datasourceKeyData[datasourceKeyData.length - 1]; + prevSeries = [prevDataEntry[0], prevDataEntry[1]]; } else { prevSeries = [0, 0]; } @@ -1178,7 +1208,7 @@ export class EntityDataSubscription { } private generateLatest(dataKey: SubscriptionDataKey, detectChanges: boolean) { - let prevSeries: [number, any]; + let prevSeries: DataEntry; const datasourceKey = dataKey.latest ? `${dataKey.key}_${dataKey.listIndex}` : dataKey.key; const datasourceKeyData = this.datasourceData[0][datasourceKey].data; if (datasourceKeyData.length > 0) { diff --git a/ui-ngx/src/app/core/api/widget-subscription.ts b/ui-ngx/src/app/core/api/widget-subscription.ts index b24e12b627..4f0f399bd0 100644 --- a/ui-ngx/src/app/core/api/widget-subscription.ts +++ b/ui-ngx/src/app/core/api/widget-subscription.ts @@ -1253,14 +1253,19 @@ export class WidgetSubscription implements IWidgetSubscription { private updateTimewindow() { this.timeWindow.interval = this.subscriptionTimewindow.aggregation.interval || 1000; this.timeWindow.timezone = this.subscriptionTimewindow.timezone; + this.timeWindow.tsOffset = this.subscriptionTimewindow.tsOffset; if (this.subscriptionTimewindow.realtimeWindowMs) { if (this.subscriptionTimewindow.quickInterval) { const startEndTime = calculateIntervalStartEndTime(this.subscriptionTimewindow.quickInterval, this.subscriptionTimewindow.timezone); this.timeWindow.maxTime = startEndTime[1] + this.subscriptionTimewindow.tsOffset; this.timeWindow.minTime = startEndTime[0] + this.subscriptionTimewindow.tsOffset; } else { - this.timeWindow.maxTime = moment().valueOf() + this.subscriptionTimewindow.tsOffset + this.timeWindow.stDiff; - this.timeWindow.minTime = this.timeWindow.maxTime - this.subscriptionTimewindow.realtimeWindowMs; + const now = moment().valueOf() + this.subscriptionTimewindow.tsOffset + this.timeWindow.stDiff; + if (!this.timeWindow.maxTime || Math.abs(now - this.timeWindow.maxTime) > 500) { + this.timeWindow.maxTime = now; + this.timeWindow.maxTime -= this.timeWindow.maxTime % 1000; + this.timeWindow.minTime = this.timeWindow.maxTime - this.subscriptionTimewindow.realtimeWindowMs; + } } } else if (this.subscriptionTimewindow.fixedWindow) { this.timeWindow.maxTime = this.subscriptionTimewindow.fixedWindow.endTimeMs + this.subscriptionTimewindow.tsOffset; diff --git a/ui-ngx/src/app/core/services/time.service.ts b/ui-ngx/src/app/core/services/time.service.ts index e9c7de1cd8..62d5a3507c 100644 --- a/ui-ngx/src/app/core/services/time.service.ts +++ b/ui-ngx/src/app/core/services/time.service.ts @@ -19,22 +19,14 @@ import { AggregationType, DAY, defaultTimeIntervals, - defaultTimewindow, + defaultTimewindow, Interval, IntervalMath, SECOND, + TimeInterval, Timewindow } from '@shared/models/time/time.models'; import { HttpClient } from '@angular/common/http'; -import { Observable } from 'rxjs'; -import { defaultHttpOptions } from '@core/http/http-utils'; -import { map } from 'rxjs/operators'; import { isDefined } from '@core/utils'; -export interface TimeInterval { - name: string; - translateParams: { [key: string]: any }; - value: number; -} - const MIN_INTERVAL = SECOND; const MAX_INTERVAL = 365 * 20 * DAY; @@ -60,15 +52,16 @@ export class TimeService { } } - public matchesExistingInterval(min: number, max: number, intervalMs: number): boolean { - const intervals = this.getIntervals(min, max); - return intervals.findIndex(interval => interval.value === intervalMs) > -1; + public matchesExistingInterval(min: number, max: number, interval: Interval, useCalendarIntervals = false): boolean { + const intervals = this.getIntervals(min, max, useCalendarIntervals); + return intervals.findIndex(timeInterval => timeInterval.value === interval) > -1; } - public getIntervals(min: number, max: number): Array { + public getIntervals(min: number, max: number, useCalendarIntervals = false): Array { min = this.boundMinInterval(min); max = this.boundMaxInterval(max); - return defaultTimeIntervals.filter((interval) => interval.value >= min && interval.value <= max); + return defaultTimeIntervals.filter((interval) => (useCalendarIntervals || typeof interval.value === 'number') && + IntervalMath.numberValue(interval.value) >= min && IntervalMath.numberValue(interval.value) <= max); } public boundMinInterval(min: number): number { @@ -85,32 +78,37 @@ export class TimeService { return this.toBound(max, MIN_INTERVAL, MAX_INTERVAL, MAX_INTERVAL); } - public boundToPredefinedInterval(min: number, max: number, intervalMs: number): number { - const intervals = this.getIntervals(min, max); + public boundToPredefinedInterval(min: number, max: number, interval: Interval, useCalendarIntervals = false): Interval { + const intervals = this.getIntervals(min, max, useCalendarIntervals); let minDelta = MAX_INTERVAL; - const boundedInterval = intervalMs || min; + const boundedInterval = interval || min; if (!intervals.length) { return boundedInterval; } - let matchedInterval: TimeInterval = intervals[0]; - intervals.forEach((interval) => { - const delta = Math.abs(interval.value - boundedInterval); - if (delta < minDelta) { - matchedInterval = interval; - minDelta = delta; - } - }); - return matchedInterval.value; + const found = intervals.find(timeInterval => timeInterval.value === boundedInterval); + if (found) { + return found.value; + } else { + let matchedInterval: TimeInterval = intervals[0]; + intervals.forEach((timeInterval) => { + const delta = Math.abs(IntervalMath.numberValue(timeInterval.value) - IntervalMath.numberValue(boundedInterval)); + if (delta <= minDelta) { + matchedInterval = timeInterval; + minDelta = delta; + } + }); + return matchedInterval.value; + } } - public boundIntervalToTimewindow(timewindow: number, intervalMs: number, aggType: AggregationType): number { + public boundIntervalToTimewindow(timewindow: number, interval: Interval, aggType: AggregationType): Interval { if (aggType === AggregationType.NONE) { return SECOND; } else { const min = this.minIntervalLimit(timewindow); const max = this.maxIntervalLimit(timewindow); - if (intervalMs) { - return this.toBound(intervalMs, min, max, intervalMs); + if (interval) { + return this.toIntervalBound(interval, min, max, interval); } else { return this.boundToPredefinedInterval(min, max, this.avgInterval(timewindow)); } @@ -154,4 +152,13 @@ export class TimeService { } } + private toIntervalBound(value: Interval, min: number, max: number, defValue: Interval): Interval { + if (isDefined(value)) { + value = IntervalMath.max(value, min); + value = IntervalMath.min(value, max); + return value; + } else { + return defValue; + } + } } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/cards/aggregated-value-card.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/cards/aggregated-value-card.models.ts index 3b2618c680..34a3433ac5 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/cards/aggregated-value-card.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/cards/aggregated-value-card.models.ts @@ -27,7 +27,7 @@ import { lastUpdateAgoDateFormat, textStyle } from '@shared/models/widget-settings.models'; -import { ComparisonResultType, DataKey, DatasourceData } from '@shared/models/widget.models'; +import { ComparisonResultType, DataEntry, DataKey, DatasourceData } from '@shared/models/widget.models'; import { DataKeyType } from '@shared/models/telemetry/telemetry.models'; import { AggregationType } from '@shared/models/time/time.models'; @@ -102,7 +102,7 @@ export const computeAggregatedCardValue = } }; -export const getTsValueByLatestDataKey = (latestData: Array, dataKey: DataKey): [number, any] => { +export const getTsValueByLatestDataKey = (latestData: Array, dataKey: DataKey): DataEntry => { if (latestData?.length) { const dsData = latestData.find(data => data.dataKey === dataKey); if (dsData?.data?.length) { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts index 3398e289d7..ea8c46880e 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts @@ -58,6 +58,7 @@ import { NamedDataSet, toNamedData } from '@home/components/widget/lib/chart/echarts-widget.models'; +import { IntervalMath } from '@shared/models/time/time.models'; interface BarChartDataItem { id: string; @@ -187,18 +188,23 @@ export class BarChartWithLabelsWidgetComponent implements OnInit, OnDestroy, Aft this.barRenderItem = (params, api) => { - const interval = this.ctx.defaultSubscription.timeWindow.interval; + const time = api.value(0) as number; + let start = api.value(2) as number; + const end = api.value(3) as number; + let interval = end - start; + if (!start || !end || !interval) { + interval = IntervalMath.numberValue(this.ctx.timeWindow.interval); + start = time - interval / 2; + } const enabledDataItems = this.dataItems.filter(d => d.enabled); const barInterval = interval / (enabledDataItems.length + 1); const intervalGap = barInterval / 2; const index = enabledDataItems.findIndex(d => d.id === params.seriesId); - const time = api.value(0) as number; const value = api.value(1); - const start = time - interval / 2; const startTime = start + intervalGap + barInterval * index; const delta = barInterval; - const lowerLeft = api.coord([startTime, value]); + const lowerLeft = api.coord([startTime, value >= 0 ? value : 0]); const height = api.size([delta, value])[1]; const width = api.size([delta, 10])[0]; @@ -256,7 +262,7 @@ export class BarChartWithLabelsWidgetComponent implements OnInit, OnDestroy, Aft this.barLabelLayoutCallback = (params) => { if (params.rect.width - params.labelRect.width < 2) { return { - y: '1000%', + y: '100000%', }; } else { return { @@ -293,28 +299,13 @@ export class BarChartWithLabelsWidgetComponent implements OnInit, OnDestroy, Aft } public onDataUpdated() { - let minTime = this.ctx.defaultSubscription.timeWindow.minTime; - let maxTime = this.ctx.defaultSubscription.timeWindow.maxTime; - let dataMin = Number.MAX_VALUE; - let dataMax = Number.MIN_VALUE; for (const item of this.dataItems) { const datasourceData = this.ctx.data ? this.ctx.data.find(d => d.dataKey === item.dataKey) : null; item.data = datasourceData?.data ? toNamedData(datasourceData.data) : []; - if (datasourceData.data.length) { - dataMin = Math.min(datasourceData.data[0][0], dataMin); - dataMax = Math.max(datasourceData.data[datasourceData.data.length-1][0], dataMax); - } - } - if (dataMin !== Number.MAX_VALUE) { - minTime = dataMin - this.ctx.defaultSubscription.timeWindow.interval / 2; - } - if (dataMax !== Number.MIN_VALUE) { - dataMax = dataMax + this.ctx.defaultSubscription.timeWindow.interval / 2; - maxTime = Math.max(dataMax, maxTime); } if (this.barChart) { - (this.barChartOptions.xAxis as any).min = minTime; - (this.barChartOptions.xAxis as any).max = maxTime; + (this.barChartOptions.xAxis as any).min = this.ctx.defaultSubscription.timeWindow.minTime; + (this.barChartOptions.xAxis as any).max = this.ctx.defaultSubscription.timeWindow.maxTime; (this.barChartOptions.xAxis as any).tbTimewindowInterval = this.ctx.defaultSubscription.timeWindow.interval; this.barChartOptions.series = this.updateSeries(); this.barChart.setOption(this.barChartOptions); @@ -332,7 +323,15 @@ export class BarChartWithLabelsWidgetComponent implements OnInit, OnDestroy, Aft color: item.dataKey.color, data: item.data, renderItem: this.barRenderItem, - labelLayout: this.barLabelLayoutCallback + labelLayout: this.barLabelLayoutCallback, + dimensions: [ + {name: 'intervalStart', type: 'number'}, + {name: 'intervalEnd', type: 'number'} + ], + encode: { + intervalStart: 2, + intervalEnd: 3 + } }; series.push(seriesOption); } @@ -407,8 +406,8 @@ export class BarChartWithLabelsWidgetComponent implements OnInit, OnDestroy, Aft axisLine: { onZero: false }, - min: this.ctx.defaultSubscription.timeWindow.minTime - this.ctx.defaultSubscription.timeWindow.interval / 2, - max: this.ctx.defaultSubscription.timeWindow.maxTime + this.ctx.defaultSubscription.timeWindow.interval / 2 + min: this.ctx.defaultSubscription.timeWindow.minTime, + max: this.ctx.defaultSubscription.timeWindow.maxTime }, yAxis: { type: 'value', @@ -425,6 +424,8 @@ export class BarChartWithLabelsWidgetComponent implements OnInit, OnDestroy, Aft if (this.settings.showTooltip) { this.barChartOptions.tooltip = { trigger: 'axis', + confine: true, + appendToBody: true, axisPointer: { type: 'shadow' }, diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/echarts-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/echarts-widget.models.ts index 23d7af8986..8bf2254a66 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/echarts-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/echarts-widget.models.ts @@ -17,7 +17,7 @@ import * as echarts from 'echarts/core'; import { Axis } from 'echarts'; import AxisModel from 'echarts/types/src/coord/cartesian/AxisModel'; -import { formatValue, isNumber } from '@core/utils'; +import { formatValue, isDefinedAndNotNull, isNumber, isString } from '@core/utils'; import TimeScale from 'echarts/types/src/scale/Time'; import { DataZoomComponent, DataZoomComponentOption, @@ -36,7 +36,8 @@ import { } from 'echarts/charts'; import { LabelLayout } from 'echarts/features'; import { CanvasRenderer, SVGRenderer } from 'echarts/renderers'; -import { DataSet } from '@shared/models/widget.models'; +import { DataEntry, DataSet } from '@shared/models/widget.models'; +import { Interval, IntervalMath } from '@shared/models/time/time.models'; import { CallbackDataParams } from 'echarts/types/dist/shared'; import { Renderer2 } from '@angular/core'; import { DateFormatProcessor, DateFormatSettings, Font } from '@shared/models/widget-settings.models'; @@ -51,13 +52,13 @@ class EChartsModule { Axis.prototype.getBandWidth = function(){ const model: AxisModel = this.model; const axisOption = model.option; - const tbTimewindowInterval = (axisOption as any).tbTimewindowInterval; - if (this.scale.type === 'time' && isNumber(tbTimewindowInterval)) { + const tbTimewindowInterval: Interval = (axisOption as any).tbTimewindowInterval; + if (this.scale.type === 'time' && (isNumber(tbTimewindowInterval) || isString(tbTimewindowInterval))) { const timeScale: TimeScale = this.scale; const axisExtent: [number, number] = this._extent; const dataExtent = timeScale.getExtent(); const size = Math.abs(axisExtent[1] - axisExtent[0]); - return tbTimewindowInterval * (size / (dataExtent[1] - dataExtent[0])); + return IntervalMath.numberValue(tbTimewindowInterval) * (size / (dataExtent[1] - dataExtent[0])); } else { return axisGetBandWidth.call(this); } @@ -99,17 +100,31 @@ export type EChartsOption = echarts.ComposeOption< export type ECharts = echarts.ECharts; -export type NamedDataSet = {name: string; value: [number, any]}[]; +export type EChartsDataItem = [number, any, number, number]; + +export type NamedDataSet = {name: string; value: EChartsDataItem}[]; export const toNamedData = (data: DataSet): NamedDataSet => { if (!data?.length) { return []; } else { - return data.map(d => ({ - name: d[0] + '', - value: d - })); + return data.map(d => { + const ts = isDefinedAndNotNull(d[2]) ? d[2][0] : d[0]; + return { + name: ts + '', + value: toEChartsDataItem(d) + }; + }); + } +}; + +const toEChartsDataItem = (entry: DataEntry): EChartsDataItem => { + const item: EChartsDataItem = [entry[0], entry[1], entry[0], entry[0]]; + if (isDefinedAndNotNull(entry[2])) { + item[2] = entry[2][0]; + item[3] = entry[2][1]; } + return item; }; export interface EChartsTooltipWidgetSettings { @@ -141,9 +156,22 @@ export const echartsTooltipFormatter = (renderer: Renderer2, renderer.setStyle(tooltipElement, 'gap', '4px'); if (settings.tooltipShowDate) { const dateElement: HTMLElement = renderer.createElement('div'); - const ts = params[0].value[0]; - tooltipDateFormat.update(ts); - renderer.appendChild(dateElement, renderer.createText(tooltipDateFormat.formatted)); + let dateText: string; + const startTs = params[0].value[2]; + const endTs = params[0].value[3]; + if (startTs && endTs && (endTs - 1) > startTs) { + const startDateText = tooltipDateFormat.update(startTs); + const endDateText = tooltipDateFormat.update(endTs - 1); + if (startDateText === endDateText) { + dateText = startDateText; + } else { + dateText = startDateText + ' - ' + endDateText; + } + } else { + const ts = params[0].value[0]; + dateText = tooltipDateFormat.update(ts); + } + renderer.appendChild(dateElement, renderer.createText(dateText)); renderer.setStyle(dateElement, 'font-family', settings.tooltipDateFont.family); renderer.setStyle(dateElement, 'font-size', settings.tooltipDateFont.size + settings.tooltipDateFont.sizeUnit); renderer.setStyle(dateElement, 'font-style', settings.tooltipDateFont.style); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.ts index 84749b146c..b199b2c64f 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.ts @@ -392,6 +392,7 @@ export class RangeChartWidgetComponent implements OnInit, OnDestroy, AfterViewIn show: false, type: 'piecewise', selected: this.selectedRanges, + dimension: 1, pieces: this.rangeItems.map(item => item.piece), outOfRange: { color: this.settings.outOfRangeColor @@ -405,6 +406,8 @@ export class RangeChartWidgetComponent implements OnInit, OnDestroy, AfterViewIn if (this.settings.showTooltip) { this.rangeChartOptions.tooltip = { trigger: 'axis', + confine: true, + appendToBody: true, formatter: (params: CallbackDataParams[]) => echartsTooltipFormatter(this.renderer, this.tooltipDateFormat, this.settings, params, this.decimals, this.units, 0), padding: [8, 12], diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.ts b/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.ts index b984ad4764..846c6b8331 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/flot-widget.ts @@ -56,7 +56,7 @@ import { } from './flot-widget.models'; import * as moment_ from 'moment'; import tinycolor from 'tinycolor2'; -import { AggregationType } from '@shared/models/time/time.models'; +import { AggregationType, IntervalMath } from '@shared/models/time/time.models'; import { CancelAnimationFrame } from '@core/services/raf.service'; import { UtilsService } from '@core/services/utils.service'; import { DataKeyType } from '@shared/models/telemetry/telemetry.models'; @@ -558,7 +558,7 @@ export class TbFlot { this.subscription.timeWindowConfig.aggregation.type === AggregationType.NONE) { this.options.series.bars.barWidth = this.defaultBarWidth; } else { - this.options.series.bars.barWidth = this.subscription.timeWindow.interval * 0.6; + this.options.series.bars.barWidth = IntervalMath.numberValue(this.subscription.timeWindow.interval) * 0.6; } } this.options.xaxes[0].min = this.subscription.timeWindow.minTime; @@ -663,7 +663,7 @@ export class TbFlot { this.subscription.timeWindowConfig.aggregation.type === AggregationType.NONE) { this.options.series.bars.barWidth = this.defaultBarWidth; } else { - this.options.series.bars.barWidth = this.subscription.timeWindow.interval * 0.6; + this.options.series.bars.barWidth = IntervalMath.numberValue(this.subscription.timeWindow.interval) * 0.6; } } @@ -681,7 +681,7 @@ export class TbFlot { this.subscription.timeWindowConfig.aggregation.type === AggregationType.NONE) { this.plot.getOptions().series.bars.barWidth = this.defaultBarWidth; } else { - this.plot.getOptions().series.bars.barWidth = this.subscription.timeWindow.interval * 0.6; + this.plot.getOptions().series.bars.barWidth = IntervalMath.numberValue(this.subscription.timeWindow.interval) * 0.6; } } this.updateData(); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/indicator/liquid-level-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/indicator/liquid-level-widget.component.ts index 738b26563f..3c79883bf3 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/indicator/liquid-level-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/indicator/liquid-level-widget.component.ts @@ -60,6 +60,7 @@ import { TranslateService } from '@ngx-translate/core'; import ITooltipsterInstance = JQueryTooltipster.ITooltipsterInstance; import { ImagePipe } from '@shared/pipe/image.pipe'; import { DomSanitizer } from '@angular/platform-browser'; +import { DataEntry } from '@shared/models/widget.models'; @Component({ selector: 'tb-liquid-level-widget', @@ -399,7 +400,7 @@ export class LiquidLevelWidgetComponent implements OnInit { return limits.min + (percentage / 100) * (limits.max - limits.min); } - private updateTooltip(value: [number, any]): void { + private updateTooltip(value: DataEntry): void { this.tooltipContent = this.getTooltipContent(value); if (this.tooltip) { @@ -494,7 +495,7 @@ export class LiquidLevelWidgetComponent implements OnInit { } } - private getTooltipContent(value?: [number, any]): string { + private getTooltipContent(value?: DataEntry): string { const contentValue = value || [0, '']; let tooltipValue: string | number = 'N/A'; diff --git a/ui-ngx/src/app/shared/components/time/timeinterval.component.html b/ui-ngx/src/app/shared/components/time/timeinterval.component.html index 704e7e9ec2..04292b4d93 100644 --- a/ui-ngx/src/app/shared/components/time/timeinterval.component.html +++ b/ui-ngx/src/app/shared/components/time/timeinterval.component.html @@ -43,7 +43,7 @@
{{ predefinedName }} - + {{ interval.name | translate:interval.translateParams }} diff --git a/ui-ngx/src/app/shared/components/time/timeinterval.component.ts b/ui-ngx/src/app/shared/components/time/timeinterval.component.ts index 2e092a3407..8775b0a6ed 100644 --- a/ui-ngx/src/app/shared/components/time/timeinterval.component.ts +++ b/ui-ngx/src/app/shared/components/time/timeinterval.component.ts @@ -16,10 +16,11 @@ import { Component, EventEmitter, forwardRef, Input, OnInit, Output } from '@angular/core'; import { ControlValueAccessor, NG_VALUE_ACCESSOR } from '@angular/forms'; -import { TimeInterval, TimeService } from '@core/services/time.service'; +import { TimeService } from '@core/services/time.service'; import { coerceNumberProperty } from '@angular/cdk/coercion'; import { SubscriptSizing } from '@angular/material/form-field'; import { coerceBoolean } from '@shared/decorators/coercion'; +import { Interval, IntervalMath, TimeInterval } from '@shared/models/time/time.models'; @Component({ selector: 'tb-timeinterval', @@ -72,6 +73,10 @@ export class TimeintervalComponent implements OnInit, ControlValueAccessor { @coerceBoolean() disabledAdvanced = false; + @Input() + @coerceBoolean() + useCalendarIntervals = false; + @Output() hideFlagChange = new EventEmitter(); @Input() disabled: boolean; @@ -84,8 +89,8 @@ export class TimeintervalComponent implements OnInit, ControlValueAccessor { mins = 1; secs = 0; - intervalMs = 0; - modelValue: number; + interval: Interval = 0; + modelValue: Interval; advanced = false; rendered = false; @@ -112,26 +117,26 @@ export class TimeintervalComponent implements OnInit, ControlValueAccessor { this.disabled = isDisabled; } - writeValue(intervalMs: number): void { - this.modelValue = intervalMs; + writeValue(interval: Interval): void { + this.modelValue = interval; this.rendered = true; if (typeof this.modelValue !== 'undefined') { const min = this.timeService.boundMinInterval(this.minValue); const max = this.timeService.boundMaxInterval(this.maxValue); - if (this.modelValue >= min && this.modelValue <= max) { - this.advanced = !this.timeService.matchesExistingInterval(this.minValue, this.maxValue, this.modelValue); - this.setIntervalMs(this.modelValue); + if (IntervalMath.numberValue(this.modelValue) >= min && IntervalMath.numberValue(this.modelValue) <= max) { + this.advanced = !this.timeService.matchesExistingInterval(this.minValue, this.maxValue, this.modelValue, this.useCalendarIntervals); + this.setInterval(this.modelValue); } else { this.boundInterval(); } } } - setIntervalMs(intervalMs: number) { + setInterval(interval: Interval) { if (!this.advanced) { - this.intervalMs = intervalMs; + this.interval = interval; } - const intervalSeconds = Math.floor(intervalMs / 1000); + const intervalSeconds = Math.floor(IntervalMath.numberValue(interval) / 1000); this.days = Math.floor(intervalSeconds / 86400); this.hours = Math.floor((intervalSeconds % 86400) / 3600); this.mins = Math.floor(((intervalSeconds % 86400) % 3600) / 60); @@ -141,19 +146,20 @@ export class TimeintervalComponent implements OnInit, ControlValueAccessor { boundInterval(updateToPreferred = false) { const min = this.timeService.boundMinInterval(this.minValue); const max = this.timeService.boundMaxInterval(this.maxValue); - this.intervals = this.timeService.getIntervals(this.minValue, this.maxValue); + this.intervals = this.timeService.getIntervals(this.minValue, this.maxValue, this.useCalendarIntervals); if (this.rendered) { - let newIntervalMs = this.modelValue; + let newInterval = this.modelValue; + const newIntervalMs = IntervalMath.numberValue(newInterval); if (newIntervalMs < min) { - newIntervalMs = min; + newInterval = min; } else if (newIntervalMs >= max && updateToPreferred) { - newIntervalMs = this.timeService.boundMaxInterval(max / 7); + newInterval = this.timeService.boundMaxInterval(max / 7); } if (!this.advanced) { - newIntervalMs = this.timeService.boundToPredefinedInterval(min, max, newIntervalMs); + newInterval = this.timeService.boundToPredefinedInterval(min, max, newInterval, this.useCalendarIntervals); } - if (newIntervalMs !== this.modelValue) { - this.setIntervalMs(newIntervalMs); + if (newInterval !== this.modelValue) { + this.setInterval(newInterval); this.updateView(); } } @@ -163,18 +169,18 @@ export class TimeintervalComponent implements OnInit, ControlValueAccessor { if (!this.rendered) { return; } - let value = null; - let intervalMs; + let value: Interval = null; + let interval: Interval; if (!this.advanced) { - intervalMs = this.intervalMs; - if (!intervalMs || isNaN(intervalMs)) { - intervalMs = this.calculateIntervalMs(); + interval = this.interval; + if (!interval || typeof interval === 'number' && isNaN(interval)) { + interval = this.calculateIntervalMs(); } } else { - intervalMs = this.calculateIntervalMs(); + interval = this.calculateIntervalMs(); } - if (!isNaN(intervalMs) && intervalMs > 0) { - value = intervalMs; + if (typeof interval === 'string' || !isNaN(interval) && interval > 0) { + value = interval; } this.modelValue = value; this.propagateChange(this.modelValue); @@ -188,19 +194,19 @@ export class TimeintervalComponent implements OnInit, ControlValueAccessor { this.secs) * 1000; } - onIntervalMsChange() { + onIntervalChange() { this.updateView(); } onAdvancedChange() { if (!this.advanced) { - this.intervalMs = this.calculateIntervalMs(); + this.interval = this.calculateIntervalMs(); } else { - let intervalMs = this.intervalMs; - if (!intervalMs || isNaN(intervalMs)) { - intervalMs = this.calculateIntervalMs(); + let interval = this.interval; + if (!interval || typeof interval === 'number' && isNaN(interval)) { + interval = this.calculateIntervalMs(); } - this.setIntervalMs(intervalMs); + this.setInterval(interval); } this.updateView(); } diff --git a/ui-ngx/src/app/shared/components/time/timewindow-panel.component.html b/ui-ngx/src/app/shared/components/time/timewindow-panel.component.html index f763cf8e85..1ad06bab3e 100644 --- a/ui-ngx/src/app/shared/components/time/timewindow-panel.component.html +++ b/ui-ngx/src/app/shared/components/time/timewindow-panel.component.html @@ -203,6 +203,7 @@ [(hideFlag)]="timewindow.hideAggInterval" (hideFlagChange)="onHideAggIntervalChanged()" [min]="minRealtimeAggInterval()" [max]="maxRealtimeAggInterval()" + useCalendarIntervals predefinedName="aggregation.group-interval"> @@ -215,6 +216,7 @@ [(hideFlag)]="timewindow.hideAggInterval" (hideFlagChange)="onHideAggIntervalChanged()" [min]="minHistoryAggInterval()" [max]="maxHistoryAggInterval()" + useCalendarIntervals predefinedName="aggregation.group-interval"> diff --git a/ui-ngx/src/app/shared/models/telemetry/telemetry.models.ts b/ui-ngx/src/app/shared/models/telemetry/telemetry.models.ts index 3531fd782a..95c8cd4925 100644 --- a/ui-ngx/src/app/shared/models/telemetry/telemetry.models.ts +++ b/ui-ngx/src/app/shared/models/telemetry/telemetry.models.ts @@ -171,6 +171,14 @@ export class AttributesSubscriptionCmd extends SubscriptionCmd { type = WsCmdType.ATTRIBUTES; } +export enum IntervalType { + MILLISECONDS = 'MILLISECONDS', + WEEK = 'WEEK', + WEEK_ISO = 'WEEK_ISO', + MONTH = 'MONTH', + QUARTER = 'QUARTER' +} + export class TimeseriesSubscriptionCmd extends SubscriptionCmd { startTs: number; timeWindow: number; @@ -197,7 +205,9 @@ export interface EntityHistoryCmd { keys: Array; startTs: number; endTs: number; + intervalType: IntervalType; interval: number; + timeZoneId: string; limit: number; agg: AggregationType; fetchLatestPreviousPoint?: boolean; @@ -211,7 +221,9 @@ export interface TimeSeriesCmd { keys: Array; startTs: number; timeWindow: number; + intervalType: IntervalType; interval: number; + timeZoneId: string; limit: number; agg: AggregationType; fetchLatestPreviousPoint?: boolean; @@ -382,12 +394,14 @@ export class TelemetryPluginCmdsWrapper implements CmdWrapper { } } +export type SubscriptionDataEntry = [number, any, number?]; + export interface SubscriptionData { - [key: string]: [number, any, number?][]; + [key: string]: SubscriptionDataEntry[]; } export interface IndexedSubscriptionData { - [id: number]: [number, any, number?][]; + [id: number]: SubscriptionDataEntry[]; } export interface SubscriptionDataHolder { diff --git a/ui-ngx/src/app/shared/models/time/time.models.ts b/ui-ngx/src/app/shared/models/time/time.models.ts index 4a063a26c3..f45fd54b5b 100644 --- a/ui-ngx/src/app/shared/models/time/time.models.ts +++ b/ui-ngx/src/app/shared/models/time/time.models.ts @@ -18,6 +18,7 @@ import { TimeService } from '@core/services/time.service'; import { deepClone, isDefined, isNumeric, isUndefined } from '@app/core/utils'; import * as moment_ from 'moment'; import * as momentTz from 'moment-timezone'; +import { IntervalType } from '@shared/models/telemetry/telemetry.models'; const moment = moment_; @@ -26,6 +27,11 @@ export const MINUTE = 60 * SECOND; export const HOUR = 60 * MINUTE; export const DAY = 24 * HOUR; export const WEEK = 7 * DAY; + +export const AVG_MONTH = Math.floor(30.44 * DAY); + +export const AVG_QUARTER = Math.floor(DAY * 365.2425 / 4); + export const YEAR = DAY * 365; export type ComparisonDuration = moment_.unitOfTime.DurationConstructor | 'previousInterval' | 'customInterval'; @@ -47,8 +53,28 @@ export enum HistoryWindowType { FOR_ALL_TIME } +export type Interval = number | IntervalType; + +export class IntervalMath { + public static max(...values: Interval[]): Interval { + const numberArr = values.map(v => IntervalMath.numberValue(v)); + const index = numberArr.indexOf(Math.max(...numberArr)); + return values[index]; + } + + public static min(...values: Interval[]): Interval { + const numberArr = values.map(v => IntervalMath.numberValue(v)); + const index = numberArr.indexOf(Math.min(...numberArr)); + return values[index]; + } + + public static numberValue(value: Interval): number { + return typeof value === 'number' ? value : IntervalTypeValuesMap.get(value); + } +} + export interface IntervalWindow { - interval?: number; + interval?: Interval; timewindowMs?: number; quickInterval?: QuickTimeInterval; } @@ -88,7 +114,7 @@ export const aggregationTranslations = new Map( ); export interface Aggregation { - interval?: number; + interval?: Interval; type: AggregationType; limit: number; } @@ -110,7 +136,7 @@ export interface Timewindow { } export interface SubscriptionAggregation extends Aggregation { - interval?: number; + interval?: Interval; timeWindow?: number; stateData?: boolean; } @@ -129,8 +155,9 @@ export interface SubscriptionTimewindow { export interface WidgetTimewindow { minTime?: number; maxTime?: number; - interval?: number; + interval?: Interval; timezone?: string; + tsOffset?: number; stDiff?: number; } @@ -188,6 +215,13 @@ export const QuickTimeIntervalTranslationMap = new Map([ + [IntervalType.WEEK, WEEK], + [IntervalType.WEEK_ISO, WEEK], + [IntervalType.MONTH, AVG_MONTH], + [IntervalType.QUARTER, AVG_QUARTER] +]); + export const forAllTimeInterval = (): Timewindow => ({ selectedTab: TimewindowType.HISTORY, history: { @@ -324,7 +358,7 @@ export const initModelFromDefaultTimewindow = (value: Timewindow, quickIntervalO }; export const toHistoryTimewindow = (timewindow: Timewindow, startTimeMs: number, endTimeMs: number, - interval: number, timeService: TimeService): Timewindow => { + interval: Interval, timeService: TimeService): Timewindow => { if (timewindow.history) { interval = isDefined(interval) ? interval : timewindow.history.interval; } else if (timewindow.realtime) { @@ -412,23 +446,23 @@ const getSubscriptionRealtimeWindowFromTimeInterval = (interval: QuickTimeInterv case QuickTimeInterval.CURRENT_MONTH: case QuickTimeInterval.CURRENT_MONTH_SO_FAR: currentDate = getCurrentTime(tz); - return currentDate.endOf('month').diff(currentDate.clone().startOf('month')); + return currentDate.endOf('month').add(1, 'milliseconds').diff(currentDate.clone().startOf('month')); case QuickTimeInterval.CURRENT_QUARTER: case QuickTimeInterval.CURRENT_QUARTER_SO_FAR: currentDate = getCurrentTime(tz); - return currentDate.endOf('quarter').diff(currentDate.clone().startOf('quarter')); + return currentDate.endOf('quarter').add(1, 'milliseconds').diff(currentDate.clone().startOf('quarter')); case QuickTimeInterval.CURRENT_HALF_YEAR: case QuickTimeInterval.CURRENT_HALF_YEAR_SO_FAR: currentDate = getCurrentTime(tz); if (currentDate.get('quarter') < 3) { - return currentDate.clone().set('quarter', 2).endOf('quarter').diff(currentDate.startOf('year')); + return currentDate.clone().set('quarter', 2).endOf('quarter').add(1, 'milliseconds').diff(currentDate.startOf('year')); } else { - return currentDate.endOf('year').diff(currentDate.clone().set('quarter', 3).startOf('quarter')); + return currentDate.endOf('year').add(1, 'milliseconds').diff(currentDate.clone().set('quarter', 3).startOf('quarter')); } case QuickTimeInterval.CURRENT_YEAR: case QuickTimeInterval.CURRENT_YEAR_SO_FAR: currentDate = getCurrentTime(tz); - return currentDate.endOf('year').diff(currentDate.clone().startOf('year')); + return currentDate.endOf('year').add(1, 'milliseconds').diff(currentDate.clone().startOf('year')); } }; @@ -551,10 +585,10 @@ export const createSubscriptionTimewindow = (timewindow: Timewindow, stDiff: num limit: timeService.getMaxDatapointsLimit(), type: AggregationType.AVG }, - timezone: timewindow.timezone, + timezone: timewindow.timezone || getDefaultTimezone(), tsOffset: calculateTsOffset(timewindow.timezone) }; - let aggTimewindow; + let aggTimewindow: number; if (stateData) { subscriptionTimewindow.aggregation.type = AggregationType.NONE; subscriptionTimewindow.aggregation.stateData = true; @@ -587,16 +621,13 @@ export const createSubscriptionTimewindow = (timewindow: Timewindow, stDiff: num subscriptionTimewindow.startTs = currentDate.valueOf() + stDiff - subscriptionTimewindow.realtimeWindowMs; } subscriptionTimewindow.aggregation.interval = - timeService.boundIntervalToTimewindow(subscriptionTimewindow.realtimeWindowMs, timewindow.realtime.interval, - subscriptionTimewindow.aggregation.type); + subscriptionTimewindow.aggregation.type === AggregationType.NONE + ? SECOND + : (!!timewindow.realtime.interval ? timewindow.realtime.interval : + timeService.boundIntervalToTimewindow(subscriptionTimewindow.realtimeWindowMs, timewindow.realtime.interval, + subscriptionTimewindow.aggregation.type)); + aggTimewindow = subscriptionTimewindow.realtimeWindowMs; - if (realtimeType !== RealtimeWindowType.INTERVAL) { - const startDiff = subscriptionTimewindow.startTs % subscriptionTimewindow.aggregation.interval; - if (startDiff) { - subscriptionTimewindow.startTs -= startDiff; - aggTimewindow += subscriptionTimewindow.aggregation.interval; - } - } } else { let historyType = timewindow.history.historyType; if (isUndefined(historyType)) { @@ -633,12 +664,17 @@ export const createSubscriptionTimewindow = (timewindow: Timewindow, stDiff: num } subscriptionTimewindow.startTs = subscriptionTimewindow.fixedWindow.startTimeMs; subscriptionTimewindow.aggregation.interval = - timeService.boundIntervalToTimewindow(aggTimewindow, timewindow.history.interval, subscriptionTimewindow.aggregation.type); + subscriptionTimewindow.aggregation.type === AggregationType.NONE + ? SECOND + : (!!timewindow.history.interval ? timewindow.history.interval : + timeService.boundIntervalToTimewindow(aggTimewindow, timewindow.history.interval, + subscriptionTimewindow.aggregation.type)); } const aggregation = subscriptionTimewindow.aggregation; aggregation.timeWindow = aggTimewindow; if (aggregation.type !== AggregationType.NONE) { - aggregation.limit = Math.ceil(aggTimewindow / subscriptionTimewindow.aggregation.interval); + aggregation.limit = calculateIntervalsCount(subscriptionTimewindow.startTs, aggTimewindow, + subscriptionTimewindow.aggregation.interval, timewindow.timezone); } return subscriptionTimewindow; }; @@ -759,12 +795,13 @@ export const createTimewindowForComparison = (subscriptionTimewindow: Subscripti fixedWindow: null, realtimeWindowMs: null, aggregation: subscriptionTimewindow.aggregation, - tsOffset: subscriptionTimewindow.tsOffset + tsOffset: subscriptionTimewindow.tsOffset, + timezone: subscriptionTimewindow.timezone }; if (subscriptionTimewindow.fixedWindow) { - let startTimeMs; - let endTimeMs; + let startTimeMs: number; + let endTimeMs: number; if (timeUnit === 'previousInterval') { if (subscriptionTimewindow.quickInterval) { const startDate = moment(subscriptionTimewindow.fixedWindow.startTimeMs); @@ -829,7 +866,7 @@ export const cloneSelectedTimewindow = (timewindow: Timewindow): Timewindow => { export interface TimeInterval { name: string; translateParams: {[key: string]: any}; - value: number; + value: Interval; } export const defaultTimeIntervals = new Array( @@ -923,10 +960,30 @@ export const defaultTimeIntervals = new Array( translateParams: {days: 7}, value: 7 * DAY }, + { + name: 'timeinterval.type.week', + translateParams: {}, + value: IntervalType.WEEK + }, + { + name: 'timeinterval.type.week-iso', + translateParams: {}, + value: IntervalType.WEEK_ISO + }, { name: 'timeinterval.days-interval', translateParams: {days: 30}, value: 30 * DAY + }, + { + name: 'timeinterval.type.month', + translateParams: {}, + value: IntervalType.MONTH + }, + { + name: 'timeinterval.type.quarter', + translateParams: {}, + value: IntervalType.QUARTER } ); @@ -1013,5 +1070,72 @@ export const getTime = (ts: number, tz?: string): moment_.Moment => { } }; +export const calculateIntervalsCount = (startTs: number, timewindow: number, interval: Interval, tz?: string): number => { + if (typeof interval === 'number') { + return Math.ceil(timewindow / interval); + } else { + const current = getTime(startTs, tz); + const endDate = getTime(startTs + timewindow, tz); + let startInterval = startIntervalDate(current, interval); + let endInterval = endIntervalDate(current, interval); + let count = 0; + while (startInterval.isBefore(endDate)) { + count++; + endInterval.add(1, 'milliseconds'); + startInterval = startIntervalDate(endInterval, interval); + endInterval = endIntervalDate(endInterval, interval); + } + return count; + } +}; + +export const startIntervalDate = (current: moment_.Moment, interval: IntervalType): moment_.Moment => { + switch (interval) { + case IntervalType.WEEK: + return current.clone().startOf('week'); + case IntervalType.WEEK_ISO: + return current.clone().startOf('isoWeek'); + case IntervalType.MONTH: + return current.clone().startOf('month'); + case IntervalType.QUARTER: + return current.clone().startOf('quarter'); + } +}; + +export const endIntervalDate = (current: moment_.Moment, interval: IntervalType): moment_.Moment => { + switch (interval) { + case IntervalType.WEEK: + return current.clone().endOf('week'); + case IntervalType.WEEK_ISO: + return current.clone().endOf('isoWeek'); + case IntervalType.MONTH: + return current.clone().endOf('month'); + case IntervalType.QUARTER: + return current.clone().endOf('quarter'); + } +}; + +export const calculateAggInterval = (subsTw: SubscriptionTimewindow, endTs: number, timestamp: number): [number, number] => { + let startIntervalTs: number; + let endIntervalTs: number; + if (typeof subsTw.aggregation.interval === 'number') { + const startTs = subsTw.startTs + subsTw.tsOffset; + startIntervalTs = startTs + Math.floor((timestamp - startTs) / subsTw.aggregation.interval) * subsTw.aggregation.interval; + endIntervalTs = startIntervalTs + subsTw.aggregation.interval; + } else { + const time = getTime(timestamp, subsTw.timezone); + let startInterval = startIntervalDate(time, subsTw.aggregation.interval); + const start = getTime(subsTw.startTs, subsTw.timezone); + if (start.isAfter(startInterval)) { + startInterval = start; + } + const endInterval = endIntervalDate(time, subsTw.aggregation.interval).add(1, 'milliseconds'); + startIntervalTs = startInterval.valueOf() + subsTw.tsOffset; + endIntervalTs = endInterval.valueOf() + subsTw.tsOffset; + } + endIntervalTs = Math.min(endIntervalTs, endTs); + return [startIntervalTs, endIntervalTs]; +}; + export const getCurrentTimeForComparison = (timeForComparison: moment_.unitOfTime.DurationConstructor, tz?: string): moment_.Moment => getCurrentTime(tz).subtract(1, timeForComparison); diff --git a/ui-ngx/src/app/shared/models/widget-settings.models.ts b/ui-ngx/src/app/shared/models/widget-settings.models.ts index 6a1cd755f6..a3d5b57954 100644 --- a/ui-ngx/src/app/shared/models/widget-settings.models.ts +++ b/ui-ngx/src/app/shared/models/widget-settings.models.ts @@ -15,7 +15,7 @@ /// import { isDefinedAndNotNull, isNumber, isNumeric, isUndefinedOrNull, parseFunction } from '@core/utils'; -import { DataKey, Datasource, DatasourceData } from '@shared/models/widget.models'; +import { DataEntry, DataKey, Datasource, DatasourceData } from '@shared/models/widget.models'; import { Injector } from '@angular/core'; import { DatePipe } from '@angular/common'; import { DateAgoPipe } from '@shared/pipe/date-ago.pipe'; @@ -366,7 +366,7 @@ export abstract class DateFormatProcessor { protected settings: DateFormatSettings) { } - abstract update(ts: string | number | Date): void; + abstract update(ts: string | number | Date): string; } @@ -380,12 +380,13 @@ export class SimpleDateFormatProcessor extends DateFormatProcessor { this.datePipe = $injector.get(DatePipe); } - update(ts: string| number | Date): void { + update(ts: string| number | Date): string { if (ts) { this.formatted = this.datePipe.transform(ts, this.settings.format); } else { this.formatted = ' '; } + return this.formatted; } } @@ -402,7 +403,7 @@ export class LastUpdateAgoDateFormatProcessor extends DateFormatProcessor { this.translate = $injector.get(TranslateService); } - update(ts: string| number | Date): void { + update(ts: string| number | Date): string { if (ts) { const agoText = this.dateAgoPipe.transform(ts, {applyAgo: true, short: true, textPart: true}); if (this.settings.hideLastUpdatePrefix) { @@ -414,6 +415,7 @@ export class LastUpdateAgoDateFormatProcessor extends DateFormatProcessor { } else { this.formatted = ' '; } + return this.formatted; } } @@ -629,7 +631,7 @@ export const setLabel = (label: string, datasources?: Datasource[]): void => { } }; -export const getSingleTsValue = (data: Array): [number, any] => { +export const getSingleTsValue = (data: Array): DataEntry => { if (data.length) { const dsData = data[0]; if (dsData.data.length) { @@ -639,7 +641,7 @@ export const getSingleTsValue = (data: Array): [number, any] => return null; }; -export const getSingleTsValueByDataKey = (data: Array, dataKey: DataKey): [number, any] => { +export const getSingleTsValueByDataKey = (data: Array, dataKey: DataKey): DataEntry => { if (data.length) { const dsData = data.find(d => d.dataKey === dataKey); if (dsData?.data?.length) { @@ -649,7 +651,7 @@ export const getSingleTsValueByDataKey = (data: Array, dataKey: return null; }; -export const getLatestSingleTsValue = (data: Array): [number, any] => { +export const getLatestSingleTsValue = (data: Array): DataEntry => { if (data.length) { const dsData = data[0]; if (dsData.data.length) { diff --git a/ui-ngx/src/app/shared/models/widget.models.ts b/ui-ngx/src/app/shared/models/widget.models.ts index 2db85b853e..c8e5699d82 100644 --- a/ui-ngx/src/app/shared/models/widget.models.ts +++ b/ui-ngx/src/app/shared/models/widget.models.ts @@ -461,7 +461,13 @@ export interface ReplaceInfo { dataKeyName: string; } -export type DataSet = [number, any][]; +export type DataEntry = [number, any, [number, number]?]; + +export type DataSet = DataEntry[]; + +export interface IndexedData { + [id: number]: DataSet; +} export interface DataSetHolder { data: DataSet; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index de8d7706d5..e0e868cf96 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -4220,6 +4220,12 @@ "current-half-year-so-far": "Current half year so far", "current-year": "Current year", "current-year-so-far": "Current year so far" + }, + "type": { + "week": "Week (Sun - Sat)", + "week-iso": "Week (Mon - Sun)", + "month": "Month", + "quarter": "Quarter" } }, "timeunit": { diff --git a/ui-ngx/yarn.lock b/ui-ngx/yarn.lock index 50815ce794..2c60a7f009 100644 --- a/ui-ngx/yarn.lock +++ b/ui-ngx/yarn.lock @@ -9840,6 +9840,11 @@ socks@^2.6.2: ip "^2.0.0" smart-buffer "^4.2.0" +sorted-btree@^1.8.1: + version "1.8.1" + resolved "https://registry.yarnpkg.com/sorted-btree/-/sorted-btree-1.8.1.tgz#6e6275f7955e5892bb8737149cbe495be10f426f" + integrity sha512-395+XIP+wqNn3USkFSrNz7G3Ss/MXlZEqesxvzCRFwL14h6e8LukDHdLBePn5pwbm5OQ9vGu8mDyz2lLDIqamQ== + "source-map-js@>=0.6.2 <2.0.0", source-map-js@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c" From d677f7c790b6ef4d77b614671a3f96057b9bcc8b Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 12 Jan 2024 16:23:24 +0200 Subject: [PATCH 5/8] UI: Fix realtime timewindow calculation for quick interval. --- ui-ngx/src/app/shared/models/time/time.models.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/shared/models/time/time.models.ts b/ui-ngx/src/app/shared/models/time/time.models.ts index f45fd54b5b..c16b109984 100644 --- a/ui-ngx/src/app/shared/models/time/time.models.ts +++ b/ui-ngx/src/app/shared/models/time/time.models.ts @@ -446,23 +446,23 @@ const getSubscriptionRealtimeWindowFromTimeInterval = (interval: QuickTimeInterv case QuickTimeInterval.CURRENT_MONTH: case QuickTimeInterval.CURRENT_MONTH_SO_FAR: currentDate = getCurrentTime(tz); - return currentDate.endOf('month').add(1, 'milliseconds').diff(currentDate.clone().startOf('month')); + return currentDate.clone().endOf('month').add(1, 'milliseconds').diff(currentDate.clone().startOf('month')); case QuickTimeInterval.CURRENT_QUARTER: case QuickTimeInterval.CURRENT_QUARTER_SO_FAR: currentDate = getCurrentTime(tz); - return currentDate.endOf('quarter').add(1, 'milliseconds').diff(currentDate.clone().startOf('quarter')); + return currentDate.clone().endOf('quarter').add(1, 'milliseconds').diff(currentDate.clone().startOf('quarter')); case QuickTimeInterval.CURRENT_HALF_YEAR: case QuickTimeInterval.CURRENT_HALF_YEAR_SO_FAR: currentDate = getCurrentTime(tz); if (currentDate.get('quarter') < 3) { - return currentDate.clone().set('quarter', 2).endOf('quarter').add(1, 'milliseconds').diff(currentDate.startOf('year')); + return currentDate.clone().set('quarter', 2).endOf('quarter').add(1, 'milliseconds').diff(currentDate.clone().startOf('year')); } else { - return currentDate.endOf('year').add(1, 'milliseconds').diff(currentDate.clone().set('quarter', 3).startOf('quarter')); + return currentDate.clone().endOf('year').add(1, 'milliseconds').diff(currentDate.clone().set('quarter', 3).startOf('quarter')); } case QuickTimeInterval.CURRENT_YEAR: case QuickTimeInterval.CURRENT_YEAR_SO_FAR: currentDate = getCurrentTime(tz); - return currentDate.endOf('year').add(1, 'milliseconds').diff(currentDate.clone().startOf('year')); + return currentDate.clone().endOf('year').add(1, 'milliseconds').diff(currentDate.clone().startOf('year')); } }; From d2ba3d2d841d085371671b97d9c825f698152d69 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 12 Jan 2024 18:03:40 +0200 Subject: [PATCH 6/8] UI: Calculate correct interval length for echarts axis pointer. --- ui-ngx/src/app/core/api/data-aggregator.ts | 6 +-- .../bar-chart-with-labels-widget.component.ts | 4 +- .../widget/lib/chart/echarts-widget.models.ts | 51 ++++++++++++++++--- .../src/app/shared/models/time/time.models.ts | 34 ++++++++----- 4 files changed, 70 insertions(+), 25 deletions(-) diff --git a/ui-ngx/src/app/core/api/data-aggregator.ts b/ui-ngx/src/app/core/api/data-aggregator.ts index a16c4baa5b..c11c08ad2b 100644 --- a/ui-ngx/src/app/core/api/data-aggregator.ts +++ b/ui-ngx/src/app/core/api/data-aggregator.ts @@ -19,7 +19,7 @@ import { IndexedSubscriptionData, } from '@app/shared/models/telemetry/telemetry.models'; import { - AggregationType, calculateAggInterval, + AggregationType, calculateAggIntervalWithSubscriptionTimeWindow, calculateIntervalComparisonEndTime, calculateIntervalEndTime, calculateIntervalStartEndTime, @@ -79,7 +79,7 @@ class AggDataMap { } calculateAggInterval(timestamp: number): [number, number] { - return calculateAggInterval(this.subsTw, this.endTs, timestamp); + return calculateAggIntervalWithSubscriptionTimeWindow(this.subsTw, this.endTs, timestamp); } updateLastInterval(endTs: number) { @@ -88,7 +88,7 @@ class AggDataMap { const lastTs = this.map.maxKey(); if (lastTs) { const data = this.map.get(lastTs); - const interval = calculateAggInterval(this.subsTw, endTs, data.ts); + const interval = calculateAggIntervalWithSubscriptionTimeWindow(this.subsTw, endTs, data.ts); data.interval = interval; data.ts = interval[0] + Math.floor((interval[1] - interval[0]) / 2); } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts index ea8c46880e..57dbf908dd 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts @@ -306,7 +306,7 @@ export class BarChartWithLabelsWidgetComponent implements OnInit, OnDestroy, Aft if (this.barChart) { (this.barChartOptions.xAxis as any).min = this.ctx.defaultSubscription.timeWindow.minTime; (this.barChartOptions.xAxis as any).max = this.ctx.defaultSubscription.timeWindow.maxTime; - (this.barChartOptions.xAxis as any).tbTimewindowInterval = this.ctx.defaultSubscription.timeWindow.interval; + (this.barChartOptions.xAxis as any).tbTimeWindow = this.ctx.defaultSubscription.timeWindow; this.barChartOptions.series = this.updateSeries(); this.barChart.setOption(this.barChartOptions); } @@ -417,7 +417,7 @@ export class BarChartWithLabelsWidgetComponent implements OnInit, OnDestroy, Aft } }; - (this.barChartOptions.xAxis as any).tbTimewindowInterval = this.ctx.defaultSubscription.timeWindow.interval; + (this.barChartOptions.xAxis as any).tbTimeWindow = this.ctx.defaultSubscription.timeWindow; this.barChartOptions.series = this.updateSeries(); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/echarts-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/echarts-widget.models.ts index 8bf2254a66..515e47559d 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/echarts-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/echarts-widget.models.ts @@ -37,7 +37,12 @@ import { import { LabelLayout } from 'echarts/features'; import { CanvasRenderer, SVGRenderer } from 'echarts/renderers'; import { DataEntry, DataSet } from '@shared/models/widget.models'; -import { Interval, IntervalMath } from '@shared/models/time/time.models'; +import { + calculateAggIntervalWithWidgetTimeWindow, + Interval, + IntervalMath, + WidgetTimewindow +} from '@shared/models/time/time.models'; import { CallbackDataParams } from 'echarts/types/dist/shared'; import { Renderer2 } from '@angular/core'; import { DateFormatProcessor, DateFormatSettings, Font } from '@shared/models/widget-settings.models'; @@ -52,13 +57,43 @@ class EChartsModule { Axis.prototype.getBandWidth = function(){ const model: AxisModel = this.model; const axisOption = model.option; - const tbTimewindowInterval: Interval = (axisOption as any).tbTimewindowInterval; - if (this.scale.type === 'time' && (isNumber(tbTimewindowInterval) || isString(tbTimewindowInterval))) { - const timeScale: TimeScale = this.scale; - const axisExtent: [number, number] = this._extent; - const dataExtent = timeScale.getExtent(); - const size = Math.abs(axisExtent[1] - axisExtent[0]); - return IntervalMath.numberValue(tbTimewindowInterval) * (size / (dataExtent[1] - dataExtent[0])); + if (this.scale.type === 'time') { + let interval: number; + const seriesDataIndices = axisOption.axisPointer?.seriesDataIndices; + if (seriesDataIndices?.length) { + const seriesDataIndex = seriesDataIndices[0]; + const series = model.ecModel.getSeriesByIndex(seriesDataIndex.seriesIndex); + if (series) { + const values = series.getData().getValues(seriesDataIndex.dataIndex); + const start = values[2]; + const end = values[3]; + if (typeof start === 'number' && typeof end === 'number') { + interval = Math.max(end - start, 1); + } + } + } + if (!interval) { + const tbTimeWindow: WidgetTimewindow = (axisOption as any).tbTimeWindow; + if (isDefinedAndNotNull(tbTimeWindow)) { + if (axisOption.axisPointer?.value && typeof axisOption.axisPointer?.value === 'number') { + const intervalArray = calculateAggIntervalWithWidgetTimeWindow(tbTimeWindow, axisOption.axisPointer.value); + const start = intervalArray[0]; + const end = intervalArray[1]; + interval = Math.max(end - start, 1); + } else { + interval = IntervalMath.numberValue(tbTimeWindow.interval); + } + } + } + if (interval) { + const timeScale: TimeScale = this.scale; + const axisExtent: [number, number] = this._extent; + const dataExtent = timeScale.getExtent(); + const size = Math.abs(axisExtent[1] - axisExtent[0]); + return interval * (size / (dataExtent[1] - dataExtent[0])); + } else { + return axisGetBandWidth.call(this); + } } else { return axisGetBandWidth.call(this); } diff --git a/ui-ngx/src/app/shared/models/time/time.models.ts b/ui-ngx/src/app/shared/models/time/time.models.ts index c16b109984..8d821b6689 100644 --- a/ui-ngx/src/app/shared/models/time/time.models.ts +++ b/ui-ngx/src/app/shared/models/time/time.models.ts @@ -1115,25 +1115,35 @@ export const endIntervalDate = (current: moment_.Moment, interval: IntervalType) } }; -export const calculateAggInterval = (subsTw: SubscriptionTimewindow, endTs: number, timestamp: number): [number, number] => { +export const calculateAggIntervalWithSubscriptionTimeWindow + = (subsTw: SubscriptionTimewindow, endTs: number, timestamp: number): [number, number] => + calculateInterval(subsTw.startTs, endTs, subsTw.aggregation.interval, subsTw.tsOffset, subsTw.timezone, timestamp); + +export const calculateAggIntervalWithWidgetTimeWindow + = (widgetTimeWindow: WidgetTimewindow, timestamp: number): [number, number] => + calculateInterval(widgetTimeWindow.minTime - widgetTimeWindow.tsOffset, + widgetTimeWindow.maxTime, widgetTimeWindow.interval, widgetTimeWindow.tsOffset, widgetTimeWindow.timezone, timestamp); + +export const calculateInterval = (startTime: number, endTime: number, + interval: Interval, tsOffset: number, timezone: string, timestamp: number): [number, number] => { let startIntervalTs: number; let endIntervalTs: number; - if (typeof subsTw.aggregation.interval === 'number') { - const startTs = subsTw.startTs + subsTw.tsOffset; - startIntervalTs = startTs + Math.floor((timestamp - startTs) / subsTw.aggregation.interval) * subsTw.aggregation.interval; - endIntervalTs = startIntervalTs + subsTw.aggregation.interval; + if (typeof interval === 'number') { + const startTs = startTime + tsOffset; + startIntervalTs = startTs + Math.floor((timestamp - startTs) / interval) * interval; + endIntervalTs = startIntervalTs + interval; } else { - const time = getTime(timestamp, subsTw.timezone); - let startInterval = startIntervalDate(time, subsTw.aggregation.interval); - const start = getTime(subsTw.startTs, subsTw.timezone); + const time = getTime(timestamp, timezone); + let startInterval = startIntervalDate(time, interval); + const start = getTime(startTime, timezone); if (start.isAfter(startInterval)) { startInterval = start; } - const endInterval = endIntervalDate(time, subsTw.aggregation.interval).add(1, 'milliseconds'); - startIntervalTs = startInterval.valueOf() + subsTw.tsOffset; - endIntervalTs = endInterval.valueOf() + subsTw.tsOffset; + const endInterval = endIntervalDate(time, interval).add(1, 'milliseconds'); + startIntervalTs = startInterval.valueOf() + tsOffset; + endIntervalTs = endInterval.valueOf() + tsOffset; } - endIntervalTs = Math.min(endIntervalTs, endTs); + endIntervalTs = Math.min(endIntervalTs, endTime); return [startIntervalTs, endIntervalTs]; }; From 0bf213be7d08d9da755622666caa78bef6d967d4 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 12 Jan 2024 18:29:24 +0200 Subject: [PATCH 7/8] UI: Improve range chart and bar chart axis pointers. --- .../bar-chart-with-labels-widget.component.ts | 40 +++++++++---------- .../lib/chart/range-chart-widget.component.ts | 31 +++++++------- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts index 57dbf908dd..2e2faf35e8 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts @@ -384,7 +384,25 @@ export class BarChartWithLabelsWidgetComponent implements OnInit, OnDestroy, Aft }); this.barChartOptions = { tooltip: { - trigger: 'none' + trigger: 'axis', + confine: true, + appendToBody: true, + axisPointer: { + type: 'shadow' + }, + formatter: (params: CallbackDataParams[]) => { + if (this.settings.showTooltip) { + const focusedSeriesIndex = this.focusedSeriesIndex(); + return echartsTooltipFormatter(this.renderer, this.tooltipDateFormat, + this.settings, params, this.decimals, this.units, focusedSeriesIndex); + } else { + return undefined; + } + }, + padding: [8, 12], + backgroundColor: this.settings.tooltipBackgroundColor, + borderWidth: 0, + extraCssText: `line-height: 1; backdrop-filter: blur(${this.settings.tooltipBackgroundBlur}px);` }, grid: { containLabel: true, @@ -421,26 +439,6 @@ export class BarChartWithLabelsWidgetComponent implements OnInit, OnDestroy, Aft this.barChartOptions.series = this.updateSeries(); - if (this.settings.showTooltip) { - this.barChartOptions.tooltip = { - trigger: 'axis', - confine: true, - appendToBody: true, - axisPointer: { - type: 'shadow' - }, - formatter: (params: CallbackDataParams[]) => { - const focusedSeriesIndex = this.focusedSeriesIndex(); - return echartsTooltipFormatter(this.renderer, this.tooltipDateFormat, - this.settings, params, this.decimals, this.units, focusedSeriesIndex); - }, - padding: [8, 12], - backgroundColor: this.settings.tooltipBackgroundColor, - borderWidth: 0, - extraCssText: `line-height: 1; backdrop-filter: blur(${this.settings.tooltipBackgroundBlur}px);` - }; - } - this.barChart.setOption(this.barChartOptions); this.shapeResize$ = new ResizeObserver(() => { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.ts index b199b2c64f..0a8c15b247 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.ts @@ -286,7 +286,8 @@ export class RangeChartWidgetComponent implements OnInit, OnDestroy, AfterViewIn this.rangeChart.setOption({ xAxis: { min: this.ctx.defaultSubscription.timeWindow.minTime, - max: this.ctx.defaultSubscription.timeWindow.maxTime + max: this.ctx.defaultSubscription.timeWindow.maxTime, + tbTimeWindow: this.ctx.defaultSubscription.timeWindow }, series: [ {data: this.ctx.data?.length ? toNamedData(this.ctx.data[0].data) : []} @@ -315,7 +316,19 @@ export class RangeChartWidgetComponent implements OnInit, OnDestroy, AfterViewIn }); this.rangeChartOptions = { tooltip: { - trigger: 'none' + trigger: 'axis', + confine: true, + appendToBody: true, + axisPointer: { + type: 'shadow' + }, + formatter: (params: CallbackDataParams[]) => + this.settings.showTooltip ? echartsTooltipFormatter(this.renderer, this.tooltipDateFormat, + this.settings, params, this.decimals, this.units, 0) : undefined, + padding: [8, 12], + backgroundColor: this.settings.tooltipBackgroundColor, + borderWidth: 0, + extraCssText: `line-height: 1; backdrop-filter: blur(${this.settings.tooltipBackgroundBlur}px);` }, grid: { containLabel: true, @@ -403,19 +416,7 @@ export class RangeChartWidgetComponent implements OnInit, OnDestroy, AfterViewIn } }; - if (this.settings.showTooltip) { - this.rangeChartOptions.tooltip = { - trigger: 'axis', - confine: true, - appendToBody: true, - formatter: (params: CallbackDataParams[]) => echartsTooltipFormatter(this.renderer, this.tooltipDateFormat, - this.settings, params, this.decimals, this.units, 0), - padding: [8, 12], - backgroundColor: this.settings.tooltipBackgroundColor, - borderWidth: 0, - extraCssText: `line-height: 1; backdrop-filter: blur(${this.settings.tooltipBackgroundBlur}px);` - }; - } + (this.rangeChartOptions.xAxis as any).tbTimeWindow = this.ctx.defaultSubscription.timeWindow; this.rangeChart.setOption(this.rangeChartOptions); From e9d86fc91877f789796d9cdc412a2ca18acb52a3 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 15 Jan 2024 13:38:15 +0200 Subject: [PATCH 8/8] Test for the precise interval end --- .../server/controller/TelemetryControllerTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/TelemetryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TelemetryControllerTest.java index 98fbffaee5..0a110ef2d0 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TelemetryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TelemetryControllerTest.java @@ -75,8 +75,10 @@ public class TelemetryControllerTest extends AbstractControllerTest { tsService.save(tenantId, device.getId(), new BasicTsKvEntry(1704899728000L, new LongDataEntry("t", 1L))); // Wednesday, January 10 15:15:28 GMT tsService.save(tenantId, device.getId(), new BasicTsKvEntry(1704899729000L, new LongDataEntry("t", 3L))); // Wednesday, January 10 15:15:29 GMT - tsService.save(tenantId, device.getId(), new BasicTsKvEntry(endOfWeek1Ts + 1000, new LongDataEntry("t", 7L))); // Monday, January 15, 2024 0:00:01 GMT+02:00 - tsService.save(tenantId, device.getId(), new BasicTsKvEntry(endOfWeek2Ts + 1000, new LongDataEntry("t", 11L))); // Monday, January 22, 2024 0:00:01 GMT+02:00 + tsService.save(tenantId, device.getId(), new BasicTsKvEntry(endOfWeek1Ts, new LongDataEntry("t", 2L))); // Monday, January 15, 2024 0:00:00 GMT+02:00 + tsService.save(tenantId, device.getId(), new BasicTsKvEntry(endOfWeek1Ts + 1000, new LongDataEntry("t", 5L))); // Monday, January 15, 2024 0:00:01 GMT+02:00 + tsService.save(tenantId, device.getId(), new BasicTsKvEntry(endOfWeek2Ts, new LongDataEntry("t", 9L))); // Monday, January 22, 2024 0:00:00 GMT+02:00 + tsService.save(tenantId, device.getId(), new BasicTsKvEntry(endOfWeek2Ts + 1000, new LongDataEntry("t", 2L))); // Monday, January 22, 2024 0:00:01 GMT+02:00 ObjectNode result = doGetAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + "/values/timeseries?keys=t&startTs={startTs}&endTs={endTs}&agg={agg}&intervalType={intervalType}&timeZone={timeZone}",