From ea88c0b251495a4844679f971a7d79439d486ef6 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 28 Oct 2025 16:43:29 +0200 Subject: [PATCH 01/40] (wip) entity aggregation CF --- ...CalculatedFieldEntityMessageProcessor.java | 14 ++- ...alculatedFieldManagerMessageProcessor.java | 11 ++ ...tractCalculatedFieldProcessingService.java | 69 +++++++++++ .../cf/CalculatedFieldProcessingService.java | 3 + ...faultCalculatedFieldProcessingService.java | 6 + .../cf/ctx/state/CalculatedFieldCtx.java | 8 ++ .../cf/ctx/state/CalculatedFieldState.java | 4 +- .../aggregation/single/AggIntervalEntry.java | 26 ++++ ...EntityAggregationCalculatedFieldState.java | 116 ++++++++++++++++++ .../utils/CalculatedFieldArgumentUtils.java | 2 + .../common/data/cf/CalculatedFieldType.java | 3 +- .../CalculatedFieldConfiguration.java | 4 +- ...gregationCalculatedFieldConfiguration.java | 59 +++++++++ .../single/interval/AggInterval.java | 44 +++++++ .../single/interval/AggIntervalType.java | 29 +++++ .../single/interval/BaseAggInterval.java | 54 ++++++++ .../single/interval/CustomInterval.java | 33 +++++ .../single/interval/DayInterval.java | 25 ++++ .../single/interval/HourInterval.java | 25 ++++ .../single/interval/MonthInterval.java | 26 ++++ .../single/interval/SpecificTimeInterval.java | 42 +++++++ .../single/interval/WeekInterval.java | 25 ++++ .../single/interval/WeekSunSatInterval.java | 25 ++++ .../single/interval/YearInterval.java | 25 ++++ 24 files changed, 672 insertions(+), 6 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/SpecificTimeInterval.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index b75946fd00..6d3116970b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -30,7 +30,9 @@ import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.EntityAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -53,6 +55,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.EntityAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -440,9 +443,14 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM GeofencingCalculatedFieldState geofencingState = (GeofencingCalculatedFieldState) state; geofencingState.updateLastDynamicArgumentsRefreshTs(); } - - Map arguments = fetchArguments(ctx); - state.update(arguments, ctx); + if (ctx.getCfType() == CalculatedFieldType.ENTITY_AGGREGATION) { + var configuration = (EntityAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + long delayUntilIntervalEnd = configuration.getInterval().getDelayUntilIntervalEnd(); + ctx.scheduleReevaluation(delayUntilIntervalEnd, actorCtx); + } else { + Map arguments = fetchArguments(ctx); + state.update(arguments, ctx); + } state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); states.put(ctx.getCfId(), state); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 4f0e323b99..65cedf443f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.EntityAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -93,6 +94,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final Map> entityIdCalculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFieldLinks = new HashMap<>(); private final Map> ownerEntities = new HashMap<>(); + private final Map entityAggCalculatedFields = new HashMap<>(); private ScheduledFuture cfsReevaluationTask; private final CalculatedFieldProcessingService cfExecService; @@ -370,6 +372,9 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } calculatedFields.put(cf.getId(), cfCtx); + if (cf.getConfiguration() instanceof EntityAggregationCalculatedFieldConfiguration aggConfig) { + entityAggCalculatedFields.put(cf.getId(), cfCtx); + } // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); @@ -401,6 +406,9 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(newCfCtx).eventEntity(newCfCtx.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(newCf.getId(), newCfCtx); + if (newCf.getConfiguration() instanceof EntityAggregationCalculatedFieldConfiguration aggConfig) { + entityAggCalculatedFields.put(newCf.getId(), newCfCtx); + } List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new CopyOnWriteArrayList<>(); boolean found = false; @@ -767,6 +775,9 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(cf.getId(), cfCtx); + if (cf.getConfiguration() instanceof EntityAggregationCalculatedFieldConfiguration aggConfig) { + entityAggCalculatedFields.put(cf.getId(), cfCtx); + } // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 945792ebcd..b1abfdf36c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -27,7 +27,11 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.EntityAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; @@ -48,14 +52,18 @@ import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntry; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.cf.CalculatedFieldType.PROPAGATION; @@ -98,6 +106,7 @@ public abstract class AbstractCalculatedFieldProcessingService { case GEOFENCING -> fetchGeofencingCalculatedFieldArguments(ctx, entityId, false, ts); case SIMPLE, SCRIPT, ALARM, PROPAGATION -> getBaseCalculatedFieldArguments(ctx, entityId, ts); case RELATED_ENTITIES_AGGREGATION -> fetchRelatedEntitiesAggArguments(ctx, entityId, ts); + case ENTITY_AGGREGATION -> null; }; if (ctx.getCfType() == PROPAGATION) { argFutures.put(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId)); @@ -117,6 +126,15 @@ public abstract class AbstractCalculatedFieldProcessingService { return futures; } + private Map> getEntityArgumentsDuringInterval(CalculatedFieldCtx ctx, EntityId entityId, long ts) { + Map> futures = new HashMap<>(); + for (var entry : ctx.getArguments().entrySet()) { + var argValueFuture = fetchArgumentValue(ctx.getTenantId(), entityId, entry.getValue(), ts); + futures.put(entry.getKey(), argValueFuture); + } + return futures; + } + protected EntityId resolveEntityId(TenantId tenantId, EntityId entityId, Argument argument) { if (argument.getRefEntityId() != null) { return argument.getRefEntityId(); @@ -276,6 +294,57 @@ public abstract class AbstractCalculatedFieldProcessingService { }; } + protected Map fetchArgumentValuesDuringInterval(EntityId entityId, AggIntervalEntry interval, CalculatedFieldCtx ctx) throws Exception { + var config = (EntityAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + Map argumentValues = new HashMap<>(); + + for (Entry entry : config.getMetrics().entrySet()) { + String metricName = entry.getKey(); + AggMetric metric = entry.getValue(); + AggFunction function = metric.getFunction(); + BaseReadTsKvQuery query = new BaseReadTsKvQuery(((AggKeyInput) metric.getInput()).getKey(), interval.getStartTs(), interval.getEndTs(), 0, 1, Aggregation.valueOf(function.name())); + log.trace("[{}][{}] Fetching timeseries for query {}", ctx.getTenantId(), entityId, query); + ListenableFuture> tsFuture = timeseriesService.findAll(ctx.getTenantId(), entityId, List.of(query)); + ListenableFuture argumentEntryFut = Futures.transform(tsFuture, timeSeries -> { + log.debug("[{}][{}] Fetched {} timeseries for query {}", ctx.getTenantId(), entityId, timeSeries == null ? 0 : timeSeries.size(), query); + if (timeSeries == null || timeSeries.isEmpty()) { + return new SingleValueArgumentEntry(); + } + return ArgumentEntry.createSingleValueArgument(timeSeries.get(0)); + }, calculatedFieldCallbackExecutor); + + // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. + // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. + // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, + // but this will significantly complicate the code. + ArgumentEntry argumentEntry = argumentEntryFut.get(1, TimeUnit.MINUTES); + argumentValues.put(metricName, argumentEntry); + } + + return argumentValues; + } + +// protected ListenableFuture fetchArgumentValuesDuringInterval(TenantId tenantId, EntityId entityId, Argument argument, AggInterval interval, long startTs) { +// return switch (argument.getRefEntityKey().getType()) { +// case ATTRIBUTE -> fetchAttribute(tenantId, entityId, argument, startTs); +// case TS_LATEST -> fetchTsLatest(tenantId, entityId, argument, startTs); +// default -> throw new IllegalStateException("Unsupported argument key type for entity aggregation calculated field: " + argument.getRefEntityKey().getType()); +// }; +// } +// +// private ListenableFuture fetchTimeSeries(TenantId tenantId, EntityId entityId, Argument argument, AggInterval interval) { +// long startInterval = System.currentTimeMillis() - interval.getIntervalDuration(); +// +// ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startInterval, System.currentTimeMillis(), 0, 1, Aggregation.NONE); +// +// log.trace("[{}][{}] Fetching timeseries for query {}", tenantId, entityId, query); +// ListenableFuture> fetchedTelemetryFut = timeseriesService.findAll(tenantId, entityId, List.of(query)); +// return Futures.transform(fetchedTelemetryFut, telemetry -> { +// log.debug("[{}][{}] Fetched {} timeseries for query {}", tenantId, entityId, telemetry == null ? 0 : telemetry.size(), query); +// return new SingleValueArgumentEntry(); +// }, calculatedFieldCallbackExecutor); +// } + private ListenableFuture fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument, long queryEndTs) { long argTimeWindow = argument.getTimeWindow() == 0 ? queryEndTs : argument.getTimeWindow(); long startInterval = queryEndTs - argTimeWindow; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index a9139572b8..3d75be44a7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntry; import java.util.List; import java.util.Map; @@ -37,6 +38,8 @@ public interface CalculatedFieldProcessingService { Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments); + Map fetchArgumentValuesDuringInterval(EntityId entityId, AggIntervalEntry interval, CalculatedFieldCtx ctx) throws Exception; + void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult result, List cfIds, TbCallback callback); void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index 52393d0ffe..d347b9701a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -47,6 +47,7 @@ import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntry; import java.util.ArrayList; import java.util.Collections; @@ -111,6 +112,11 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF return resolveArgumentFutures(argFutures); } + @Override + public Map fetchArgumentValuesDuringInterval(EntityId entityId, AggIntervalEntry interval, CalculatedFieldCtx ctx) throws Exception { + return super.fetchArgumentValuesDuringInterval(entityId, interval, ctx); + } + @Override public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult result, List cfIds, TbCallback callback) { if (!(result instanceof PropagationCalculatedFieldResult propagationCalculatedFieldResult)) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 927787eae1..6245435b92 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -45,6 +45,7 @@ import org.thingsboard.server.common.data.cf.configuration.ScheduledUpdateSuppor import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.EntityAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -56,7 +57,9 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; @@ -98,6 +101,7 @@ public class CalculatedFieldCtx implements Closeable { private TbelInvokeService tbelInvokeService; private RelationService relationService; private AlarmSubscriptionService alarmService; + private CalculatedFieldProcessingService cfProcessingService; private Map tbelExpressions; private Map> simpleExpressions; @@ -190,6 +194,9 @@ public class CalculatedFieldCtx implements Closeable { if (calculatedField.getConfiguration() instanceof ScheduledUpdateSupportedCalculatedFieldConfiguration scheduledConfig) { this.scheduledUpdateIntervalMillis = scheduledConfig.isScheduledUpdateEnabled() ? TimeUnit.SECONDS.toMillis(scheduledConfig.getScheduledUpdateInterval()) : -1L; } + if (calculatedField.getConfiguration() instanceof EntityAggregationCalculatedFieldConfiguration entityAggregationConfig) { + this.scheduledUpdateIntervalMillis = entityAggregationConfig.getInterval().getIntervalDuration(); + } this.requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); if (calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig) { this.useLatestTs = aggConfig.isUseLatestTs(); @@ -198,6 +205,7 @@ public class CalculatedFieldCtx implements Closeable { this.tbelInvokeService = systemContext.getTbelInvokeService(); this.relationService = systemContext.getRelationService(); this.alarmService = systemContext.getAlarmService(); + this.cfProcessingService = systemContext.getCalculatedFieldProcessingService(); this.maxDataPointsPerRollingArg = systemContext.getApiLimitService().getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); // fixme why tenant profile update is not handled?? this.maxStateSize = systemContext.getApiLimitService().getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 9598cc2b49..0872d98ec0 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.EntityAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; @@ -44,7 +45,8 @@ import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArg @Type(value = GeofencingCalculatedFieldState.class, name = "GEOFENCING"), @Type(value = AlarmCalculatedFieldState.class, name = "ALARM"), @Type(value = PropagationCalculatedFieldState.class, name = "PROPAGATION"), - @Type(value = RelatedEntitiesAggregationCalculatedFieldState.class, name = "RELATED_ENTITIES_AGGREGATION") + @Type(value = RelatedEntitiesAggregationCalculatedFieldState.class, name = "RELATED_ENTITIES_AGGREGATION"), + @Type(value = EntityAggregationCalculatedFieldState.class, name = "ENTITY_AGGREGATION") }) public interface CalculatedFieldState extends Closeable { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntry.java new file mode 100644 index 0000000000..657cd002ac --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntry.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.single; + +import lombok.Data; + +@Data +public class AggIntervalEntry { + + public Long startTs; + public Long endTs; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java new file mode 100644 index 0000000000..6349c635e0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java @@ -0,0 +1,116 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.single; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.Setter; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.actors.TbActorRef; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.EntityAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; +import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.BaseCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.HashMap; +import java.util.Map; + +import static java.util.concurrent.TimeUnit.SECONDS; + +public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldState { + + private Map aggIntervals = new HashMap<>(); + + @Setter + private long lastArgsRefreshTs = -1; + @Setter + private long lastMetricsEvalTs = -1; + + private long deduplicationIntervalMs = -1; + + CalculatedFieldProcessingService cfProcessingService; + + public EntityAggregationCalculatedFieldState(EntityId entityId) { + super(entityId); + } + + @Override + public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) { + super.setCtx(ctx, actorCtx); + this.cfProcessingService = ctx.getCfProcessingService(); + var configuration = (EntityAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + deduplicationIntervalMs = SECONDS.toMillis(configuration.getDeduplicationIntervalInSec()); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.ENTITY_AGGREGATION; + } + + @Override + public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception { + long endTs = System.currentTimeMillis(); + long startTs = endTs - 1000; + AggIntervalEntry interval = new AggIntervalEntry(); + + Map metrics = cfProcessingService.fetchArgumentValuesDuringInterval(entityId, interval, ctx); + + Output output = ctx.getOutput(); + lastMetricsEvalTs = System.currentTimeMillis(); + ctx.scheduleReevaluation(deduplicationIntervalMs, actorCtx); + ObjectNode result = toResult(endTs, metrics); + if (result != null) { + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .type(output.getType()) + .scope(output.getScope()) + .result(result) + .build()); + } + return Futures.immediateFuture(TelemetryCalculatedFieldResult.EMPTY); + } + + + protected ObjectNode toResult(long endTs, Map metrics) { + ObjectNode metricsNode = JacksonUtil.newObjectNode(); + for (Map.Entry entry : metrics.entrySet()) { + String metricName = entry.getKey(); + ArgumentEntry argumentEntry = entry.getValue(); + if (!argumentEntry.isEmpty()) { + metricsNode.put(metricName, JacksonUtil.toString(argumentEntry.getValue())); + } + } + ObjectNode resultNode = JacksonUtil.newObjectNode(); + if (!metricsNode.isEmpty()) { + resultNode.put("ts", endTs); + resultNode.set("values", metricsNode); + } + return resultNode; + } + + + @Override + public boolean isReady() { + return true; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index 0c0d401688..72b5b73471 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -34,6 +34,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.EntityAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -83,6 +84,7 @@ public class CalculatedFieldArgumentUtils { case ALARM -> new AlarmCalculatedFieldState(entityId); case PROPAGATION -> new PropagationCalculatedFieldState(entityId); case RELATED_ENTITIES_AGGREGATION -> new RelatedEntitiesAggregationCalculatedFieldState(entityId); + case ENTITY_AGGREGATION -> new EntityAggregationCalculatedFieldState(entityId); }; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index 4463c835db..de36b43a40 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -26,7 +26,8 @@ public enum CalculatedFieldType { GEOFENCING, ALARM, PROPAGATION, - RELATED_ENTITIES_AGGREGATION; + RELATED_ENTITIES_AGGREGATION, + ENTITY_AGGREGATION; public static final Set all = Collections.unmodifiableSet(EnumSet.allOf(CalculatedFieldType.class)); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 3df9a32dcc..0ab402ff91 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.EntityAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -42,7 +43,8 @@ import java.util.stream.Collectors; @Type(value = GeofencingCalculatedFieldConfiguration.class, name = "GEOFENCING"), @Type(value = AlarmCalculatedFieldConfiguration.class, name = "ALARM"), @Type(value = PropagationCalculatedFieldConfiguration.class, name = "PROPAGATION"), - @Type(value = RelatedEntitiesAggregationCalculatedFieldConfiguration.class, name = "RELATED_ENTITIES_AGGREGATION") + @Type(value = RelatedEntitiesAggregationCalculatedFieldConfiguration.class, name = "RELATED_ENTITIES_AGGREGATION"), + @Type(value = EntityAggregationCalculatedFieldConfiguration.class, name = "ENTITY_AGGREGATION") }) @JsonIgnoreProperties(ignoreUnknown = true) public interface CalculatedFieldConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..69f4b30d46 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation.single; + +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentsBasedCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.AggInterval; + +import java.util.Map; + +@Data +public class EntityAggregationCalculatedFieldConfiguration implements ArgumentsBasedCalculatedFieldConfiguration { + + private Map arguments; + @Valid + @NotEmpty + private Map metrics; + + private AggInterval interval; + private long deduplicationIntervalInSec; + private long watermark; + + private Output output; + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.ENTITY_AGGREGATION; + } + + @Override + public void validate() { + if (arguments.containsKey("ctx")) { + throw new IllegalArgumentException("Argument name 'ctx' is reserved and cannot be used."); + } + if (arguments.values().stream().anyMatch(Argument::hasTsRollingArgument)) { + throw new IllegalArgumentException("Calculated field with type: '" + getType() + "' doesn't support TS_ROLLING arguments."); + } + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java new file mode 100644 index 0000000000..cf6f825560 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = HourInterval.class, name = "HOUR"), + @JsonSubTypes.Type(value = DayInterval.class, name = "DAY"), + @JsonSubTypes.Type(value = WeekInterval.class, name = "WEEK"), + @JsonSubTypes.Type(value = WeekSunSatInterval.class, name = "WEEK_SUN_SAT"), + @JsonSubTypes.Type(value = MonthInterval.class, name = "MONTH"), + @JsonSubTypes.Type(value = YearInterval.class, name = "YEAR"), + @JsonSubTypes.Type(value = CustomInterval.class, name = "CUSTOM"), + @JsonSubTypes.Type(value = SpecificTimeInterval.class, name = "SPECIFIC_TIME"), +}) +@JsonIgnoreProperties(ignoreUnknown = true) +public interface AggInterval { + + AggIntervalType getType(); + + long getDelayUntilIntervalEnd(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java new file mode 100644 index 0000000000..a5c29b4edf --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; + +public enum AggIntervalType { + + HOUR, + DAY, + WEEK, + WEEK_SUN_SAT, + MONTH, + YEAR, + CUSTOM, + SPECIFIC_TIME + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java new file mode 100644 index 0000000000..5790d7545d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; + +import java.time.DayOfWeek; +import java.time.Duration; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; + +public abstract class BaseAggInterval implements AggInterval { + + protected long offsetMillis; // delay millis since start of interval + + @Override + public long getDelayUntilIntervalEnd() { + return getDelayUntilIntervalEnd(getType(), 1); + } + + protected long getDelayUntilIntervalEnd(AggIntervalType type, long multiplier) { + ZonedDateTime now = ZonedDateTime.now(); + ZonedDateTime next; + + switch (getType()) { + case HOUR -> next = now.plusHours(multiplier).truncatedTo(ChronoUnit.HOURS); + case DAY -> next = now.plusDays(multiplier).truncatedTo(ChronoUnit.DAYS); + case WEEK -> next = now.plusWeeks(multiplier).with(DayOfWeek.MONDAY).truncatedTo(ChronoUnit.DAYS); + case WEEK_SUN_SAT -> next = now.plusWeeks(multiplier).with(DayOfWeek.SUNDAY).truncatedTo(ChronoUnit.DAYS); + case MONTH -> next = now.plusMonths(multiplier).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS); + case YEAR -> next = now.plusYears(multiplier).withDayOfYear(1).truncatedTo(ChronoUnit.DAYS); + default -> throw new IllegalArgumentException("Unsupported type: " + getType()); + } + + long delayMillis = Duration.between(now, next).toMillis(); + if (offsetMillis > 0) { + delayMillis += offsetMillis; + } + + return Math.max(delayMillis, 0); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java new file mode 100644 index 0000000000..585e11d955 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; + +public class CustomInterval extends BaseAggInterval { + + private int multiplier; // number of base units (e.g. 2 hours, 5 days) + private AggIntervalType internalIntervalType; + + @Override + public AggIntervalType getType() { + return AggIntervalType.CUSTOM; + } + + @Override + public long getDelayUntilIntervalEnd() { + return super.getDelayUntilIntervalEnd(internalIntervalType, multiplier); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java new file mode 100644 index 0000000000..19620c6fda --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; + +public class DayInterval extends BaseAggInterval { + + @Override + public AggIntervalType getType() { + return AggIntervalType.DAY; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java new file mode 100644 index 0000000000..303ff8305e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; + +public class HourInterval extends BaseAggInterval { + + @Override + public AggIntervalType getType() { + return AggIntervalType.HOUR; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java new file mode 100644 index 0000000000..1d7eaf14bc --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; + +public class MonthInterval extends BaseAggInterval { + + @Override + public AggIntervalType getType() { + return AggIntervalType.MONTH; + } + + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/SpecificTimeInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/SpecificTimeInterval.java new file mode 100644 index 0000000000..2b44366f8b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/SpecificTimeInterval.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; + +import java.time.LocalTime; + +public class SpecificTimeInterval implements AggInterval { + + public Long startMillis; // start millis since start of day + public Long endMillis; // end millis since start of day + + @Override + public AggIntervalType getType() { + return AggIntervalType.SPECIFIC_TIME; + } + + @Override + public long getDelayUntilIntervalEnd() { + long nowMillis = LocalTime.now().toNanoOfDay() / 1_000_000L; + long delayMillis; + if (nowMillis < endMillis) { + delayMillis = endMillis - nowMillis; // later today + } else { + delayMillis = (24 * 60 * 60 * 1000L - nowMillis) + endMillis; // next day + } + return delayMillis; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java new file mode 100644 index 0000000000..4dd10a2747 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; + +public class WeekInterval extends BaseAggInterval { + + @Override + public AggIntervalType getType() { + return AggIntervalType.WEEK; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java new file mode 100644 index 0000000000..c12e7dc584 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; + +public class WeekSunSatInterval extends BaseAggInterval { + + @Override + public AggIntervalType getType() { + return AggIntervalType.WEEK_SUN_SAT; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java new file mode 100644 index 0000000000..24c6c72ed1 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; + +public class YearInterval extends BaseAggInterval { + + @Override + public AggIntervalType getType() { + return AggIntervalType.YEAR; + } + +} From 322f0b444dbd026499dfdee8c1fd9a3e5b5c0a95 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 29 Oct 2025 16:56:33 +0200 Subject: [PATCH 02/40] implemented state --- .../server/actors/ActorSystemContext.java | 4 +- ...CalculatedFieldEntityMessageProcessor.java | 7 +- ...alculatedFieldManagerMessageProcessor.java | 2 +- ...tractCalculatedFieldProcessingService.java | 102 +++++++--- .../cf/CalculatedFieldProcessingService.java | 4 +- ...faultCalculatedFieldProcessingService.java | 9 +- .../cf/ctx/state/ArgumentEntryType.java | 2 +- .../cf/ctx/state/CalculatedFieldCtx.java | 23 ++- .../aggregation/single/AggIntervalEntry.java | 10 +- .../single/AggIntervalEntryStatus.java | 26 +++ .../EntityAggregationArgumentEntry.java | 59 ++++++ ...EntityAggregationCalculatedFieldState.java | 184 ++++++++++++++---- .../server/utils/CalculatedFieldUtils.java | 1 + .../src/main/resources/thingsboard.yml | 5 +- .../thingsboard/server/cf/AlarmRulesTest.java | 2 +- ...gregationCalculatedFieldConfiguration.java | 7 - ...gregationCalculatedFieldConfiguration.java | 4 +- .../single/interval/AggInterval.java | 9 +- .../single/interval/AggIntervalType.java | 3 +- .../single/interval/BaseAggInterval.java | 122 ++++++++++-- .../single/interval/CustomInterval.java | 15 ++ .../single/interval/DayInterval.java | 3 + .../single/interval/HourInterval.java | 3 + .../single/interval/MonthInterval.java | 3 + .../single/interval/SpecificTimeInterval.java | 42 ---- .../single/interval/Watermark.java | 11 ++ .../single/interval/WeekInterval.java | 3 + .../single/interval/WeekSunSatInterval.java | 3 + .../single/interval/YearInterval.java | 3 + 29 files changed, 507 insertions(+), 164 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntryStatus.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationArgumentEntry.java delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/SpecificTimeInterval.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/Watermark.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 35cf9cb467..c8b4c37ead 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -664,9 +664,9 @@ public class ActorSystemContext { @Getter private long cfCalculationResultTimeout; - @Value("${actors.alarms.reevaluation_interval:120}") + @Value("${actors.calculated_fields.check_interval:120}") @Getter - private long alarmRulesReevaluationInterval; + private long cfCheckInterval; @Autowired @Getter diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 6d3116970b..083af60945 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -30,7 +30,6 @@ import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.aggregation.single.EntityAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -55,7 +54,6 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; -import org.thingsboard.server.service.cf.ctx.state.aggregation.single.EntityAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -447,10 +445,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM var configuration = (EntityAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); long delayUntilIntervalEnd = configuration.getInterval().getDelayUntilIntervalEnd(); ctx.scheduleReevaluation(delayUntilIntervalEnd, actorCtx); - } else { - Map arguments = fetchArguments(ctx); - state.update(arguments, ctx); } + Map arguments = fetchArguments(ctx); + state.update(arguments, ctx); state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); states.put(ctx.getCfId(), state); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 65cedf443f..075918e6e8 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -186,7 +186,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } catch (Exception e) { log.warn("[{}] Failed to trigger CFs reevaluation", tenantId, e); } - }, systemContext.getAlarmRulesReevaluationInterval(), systemContext.getAlarmRulesReevaluationInterval(), TimeUnit.SECONDS); + }, systemContext.getCfCheckInterval(), systemContext.getCfCheckInterval(), TimeUnit.SECONDS); } public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) throws CalculatedFieldException { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index b1abfdf36c..2110a78f5c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInp import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.aggregation.single.EntityAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.AggInterval; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; @@ -39,6 +40,7 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -53,6 +55,8 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntryStatus; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.EntityAggregationArgumentEntry; import java.util.Collections; import java.util.HashMap; @@ -63,7 +67,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.cf.CalculatedFieldType.PROPAGATION; @@ -106,7 +109,7 @@ public abstract class AbstractCalculatedFieldProcessingService { case GEOFENCING -> fetchGeofencingCalculatedFieldArguments(ctx, entityId, false, ts); case SIMPLE, SCRIPT, ALARM, PROPAGATION -> getBaseCalculatedFieldArguments(ctx, entityId, ts); case RELATED_ENTITIES_AGGREGATION -> fetchRelatedEntitiesAggArguments(ctx, entityId, ts); - case ENTITY_AGGREGATION -> null; + case ENTITY_AGGREGATION -> fetchEntityAggArguments(ctx, entityId, ts); }; if (ctx.getCfType() == PROPAGATION) { argFutures.put(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId)); @@ -201,6 +204,16 @@ public abstract class AbstractCalculatedFieldProcessingService { )); } + protected Map> fetchEntityAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { + EntityAggregationCalculatedFieldConfiguration aggConfig = (EntityAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + + return aggConfig.getArguments().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> fetchTimeSeries(ctx.getTenantId(), entityId, entry.getValue(), aggConfig.getInterval()) + )); + } + private ListenableFuture> resolveRelatedEntities(TenantId tenantId, EntityId entityId, RelationPathLevel relation) { ListenableFuture> relationsFut = relationService.findByRelationPathQueryAsync(tenantId, new EntityRelationPathQuery(entityId, List.of(relation))); @@ -294,15 +307,21 @@ public abstract class AbstractCalculatedFieldProcessingService { }; } - protected Map fetchArgumentValuesDuringInterval(EntityId entityId, AggIntervalEntry interval, CalculatedFieldCtx ctx) throws Exception { + protected Map fetchMetricsDuringInterval(EntityId entityId, AggIntervalEntry interval, CalculatedFieldCtx ctx) throws Exception { var config = (EntityAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - Map argumentValues = new HashMap<>(); + Map metricsResult = new HashMap<>(); for (Entry entry : config.getMetrics().entrySet()) { String metricName = entry.getKey(); AggMetric metric = entry.getValue(); AggFunction function = metric.getFunction(); - BaseReadTsKvQuery query = new BaseReadTsKvQuery(((AggKeyInput) metric.getInput()).getKey(), interval.getStartTs(), interval.getEndTs(), 0, 1, Aggregation.valueOf(function.name())); + + AggKeyInput input = (AggKeyInput) metric.getInput(); + String argName = input.getKey(); + Argument argument = ctx.getArguments().get(argName); + String key = argument.getRefEntityKey().getKey(); + + BaseReadTsKvQuery query = new BaseReadTsKvQuery(key, interval.getStartTs(), interval.getEndTs(), 0, 1, Aggregation.valueOf(function.name())); log.trace("[{}][{}] Fetching timeseries for query {}", ctx.getTenantId(), entityId, query); ListenableFuture> tsFuture = timeseriesService.findAll(ctx.getTenantId(), entityId, List.of(query)); ListenableFuture argumentEntryFut = Futures.transform(tsFuture, timeSeries -> { @@ -318,32 +337,61 @@ public abstract class AbstractCalculatedFieldProcessingService { // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, // but this will significantly complicate the code. ArgumentEntry argumentEntry = argumentEntryFut.get(1, TimeUnit.MINUTES); - argumentValues.put(metricName, argumentEntry); + metricsResult.put(metricName, argumentEntry); } - return argumentValues; + return metricsResult; + } + + protected ArgumentEntry fetchMetricDuringInterval(EntityId entityId, AggIntervalEntry interval, String metricName, CalculatedFieldCtx ctx) throws Exception { + var config = (EntityAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + + AggMetric metric = config.getMetrics().get(metricName); + AggFunction function = metric.getFunction(); + + AggKeyInput input = (AggKeyInput) metric.getInput(); + String argName = input.getKey(); + Argument argument = ctx.getArguments().get(argName); + String key = argument.getRefEntityKey().getKey(); + + BaseReadTsKvQuery query = new BaseReadTsKvQuery(key, interval.getStartTs(), interval.getEndTs(), 0, 1, Aggregation.valueOf(function.name())); + log.trace("[{}][{}] Fetching timeseries for query {}", ctx.getTenantId(), entityId, query); + ListenableFuture> tsFuture = timeseriesService.findAll(ctx.getTenantId(), entityId, List.of(query)); + ListenableFuture argumentEntryFut = Futures.transform(tsFuture, timeSeries -> { + log.debug("[{}][{}] Fetched {} timeseries for query {}", ctx.getTenantId(), entityId, timeSeries == null ? 0 : timeSeries.size(), query); + if (timeSeries == null || timeSeries.isEmpty()) { + return new SingleValueArgumentEntry(); + } + return ArgumentEntry.createSingleValueArgument(timeSeries.get(0)); + }, calculatedFieldCallbackExecutor); + + // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. + // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. + // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, + // but this will significantly complicate the code. + return argumentEntryFut.get(1, TimeUnit.MINUTES); } -// protected ListenableFuture fetchArgumentValuesDuringInterval(TenantId tenantId, EntityId entityId, Argument argument, AggInterval interval, long startTs) { -// return switch (argument.getRefEntityKey().getType()) { -// case ATTRIBUTE -> fetchAttribute(tenantId, entityId, argument, startTs); -// case TS_LATEST -> fetchTsLatest(tenantId, entityId, argument, startTs); -// default -> throw new IllegalStateException("Unsupported argument key type for entity aggregation calculated field: " + argument.getRefEntityKey().getType()); -// }; -// } -// -// private ListenableFuture fetchTimeSeries(TenantId tenantId, EntityId entityId, Argument argument, AggInterval interval) { -// long startInterval = System.currentTimeMillis() - interval.getIntervalDuration(); -// -// ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startInterval, System.currentTimeMillis(), 0, 1, Aggregation.NONE); -// -// log.trace("[{}][{}] Fetching timeseries for query {}", tenantId, entityId, query); -// ListenableFuture> fetchedTelemetryFut = timeseriesService.findAll(tenantId, entityId, List.of(query)); -// return Futures.transform(fetchedTelemetryFut, telemetry -> { -// log.debug("[{}][{}] Fetched {} timeseries for query {}", tenantId, entityId, telemetry == null ? 0 : telemetry.size(), query); -// return new SingleValueArgumentEntry(); -// }, calculatedFieldCallbackExecutor); -// } + private ListenableFuture fetchTimeSeries(TenantId tenantId, EntityId entityId, Argument argument, AggInterval interval) { + long startInterval = interval.getCurrentIntervalStartTs(); + + String key = argument.getRefEntityKey().getKey(); + ReadTsKvQuery query = new BaseReadTsKvQuery(key, startInterval, System.currentTimeMillis(), 0, 1, Aggregation.NONE); + + log.trace("[{}][{}] Fetching timeseries for query {}", tenantId, entityId, query); + ListenableFuture> fetchedTelemetryFut = timeseriesService.findAll(tenantId, entityId, List.of(query)); + return Futures.transform(fetchedTelemetryFut, telemetry -> { + log.debug("[{}][{}] Fetched {} timeseries for query {}", tenantId, entityId, telemetry == null ? 0 : telemetry.size(), query); + Map aggIntervals = new HashMap<>(); + AggIntervalEntry aggIntervalEntry = new AggIntervalEntry(interval.getCurrentIntervalStartTs(), interval.getCurrentIntervalEndTs()); + if (telemetry == null || telemetry.isEmpty()) { + aggIntervals.put(aggIntervalEntry, new AggIntervalEntryStatus()); + } else { + aggIntervals.put(aggIntervalEntry, new AggIntervalEntryStatus(System.currentTimeMillis())); + } + return new EntityAggregationArgumentEntry(aggIntervals); + }, calculatedFieldCallbackExecutor); + } private ListenableFuture fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument, long queryEndTs) { long argTimeWindow = argument.getTimeWindow() == 0 ? queryEndTs : argument.getTimeWindow(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index 3d75be44a7..54796e18eb 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -38,7 +38,9 @@ public interface CalculatedFieldProcessingService { Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments); - Map fetchArgumentValuesDuringInterval(EntityId entityId, AggIntervalEntry interval, CalculatedFieldCtx ctx) throws Exception; + Map fetchMetricsDuringInterval(EntityId entityId, AggIntervalEntry interval, CalculatedFieldCtx ctx) throws Exception; + + ArgumentEntry fetchMetricDuringInterval(EntityId entityId, AggIntervalEntry interval, String argName, CalculatedFieldCtx ctx) throws Exception; void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult result, List cfIds, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index d347b9701a..35f767fde2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -113,8 +113,13 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF } @Override - public Map fetchArgumentValuesDuringInterval(EntityId entityId, AggIntervalEntry interval, CalculatedFieldCtx ctx) throws Exception { - return super.fetchArgumentValuesDuringInterval(entityId, interval, ctx); + public Map fetchMetricsDuringInterval(EntityId entityId, AggIntervalEntry interval, CalculatedFieldCtx ctx) throws Exception { + return super.fetchMetricsDuringInterval(entityId, interval, ctx); + } + + @Override + public ArgumentEntry fetchMetricDuringInterval(EntityId entityId, AggIntervalEntry interval, String metricName, CalculatedFieldCtx ctx) throws Exception { + return super.fetchMetricDuringInterval(entityId, interval, metricName, ctx); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java index 427df2bf5b..457dd79686 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java @@ -16,5 +16,5 @@ package org.thingsboard.server.service.cf.ctx.state; public enum ArgumentEntryType { - SINGLE_VALUE, TS_ROLLING, GEOFENCING, PROPAGATION, RELATED_ENTITIES + SINGLE_VALUE, TS_ROLLING, GEOFENCING, PROPAGATION, RELATED_ENTITIES, ENTITY_AGGREGATION } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 6245435b92..0405687b4c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.cf.configuration.AlarmCalculatedFieldC import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ArgumentsBasedCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ExpressionBasedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration; @@ -46,6 +47,7 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.aggregation.single.EntityAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.AggInterval; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -96,6 +98,8 @@ public class CalculatedFieldCtx implements Closeable { private String expression; private boolean useLatestTs; private boolean requiresScheduledReevaluation; +// +// private long lastReevaluationTs; private ActorSystemContext systemContext; private TbelInvokeService tbelInvokeService; @@ -194,9 +198,6 @@ public class CalculatedFieldCtx implements Closeable { if (calculatedField.getConfiguration() instanceof ScheduledUpdateSupportedCalculatedFieldConfiguration scheduledConfig) { this.scheduledUpdateIntervalMillis = scheduledConfig.isScheduledUpdateEnabled() ? TimeUnit.SECONDS.toMillis(scheduledConfig.getScheduledUpdateInterval()) : -1L; } - if (calculatedField.getConfiguration() instanceof EntityAggregationCalculatedFieldConfiguration entityAggregationConfig) { - this.scheduledUpdateIntervalMillis = entityAggregationConfig.getInterval().getIntervalDuration(); - } this.requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); if (calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig) { this.useLatestTs = aggConfig.isUseLatestTs(); @@ -211,6 +212,21 @@ public class CalculatedFieldCtx implements Closeable { this.maxStateSize = systemContext.getApiLimitService().getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024; this.maxSingleValueArgumentSize = systemContext.getApiLimitService().getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxSingleValueArgumentSizeInKBytes) * 1024; } +// +// public boolean isRequiresScheduledReevaluation() { +// if (CalculatedFieldType.ENTITY_AGGREGATION.equals(calculatedField.getType())) { +// var configuration = (EntityAggregationCalculatedFieldConfiguration) calculatedField.getConfiguration(); +// AggInterval interval = configuration.getInterval(); +// long delayUntilIntervalEnd = interval.getDelayUntilIntervalEnd(); +// if (lastReevaluationTs < System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(systemContext.getCfCheckInterval())) { +// +// } +// if (TimeUnit.SECONDS.toMillis(systemContext.getCfCheckInterval()) >= delayUntilIntervalEnd) { +// return true; +// } +// } +// return requiresScheduledReevaluation; +// } public void init() { switch (cfType) { @@ -252,6 +268,7 @@ public class CalculatedFieldCtx implements Closeable { }); initialized = true; } + case ENTITY_AGGREGATION -> initialized = true; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntry.java index 657cd002ac..fac4e403a8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntry.java @@ -15,12 +15,18 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation.single; +import lombok.AllArgsConstructor; import lombok.Data; @Data +@AllArgsConstructor public class AggIntervalEntry { - public Long startTs; - public Long endTs; + private Long startTs; + private Long endTs; + + public boolean belongsToInterval(long ts) { + return ts >= startTs && ts <= endTs; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntryStatus.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntryStatus.java new file mode 100644 index 0000000000..09fa341961 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntryStatus.java @@ -0,0 +1,26 @@ +package org.thingsboard.server.service.cf.ctx.state.aggregation.single; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Data +@NoArgsConstructor +public class AggIntervalEntryStatus { + + @Setter + private long lastArgsRefreshTs = -1; + @Setter + private long lastMetricsEvalTs = -1; + + public AggIntervalEntryStatus(long lastArgsRefreshTs) { + this.lastArgsRefreshTs = lastArgsRefreshTs; + } + + public boolean shouldRecalculate(long checkInterval) { + boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - checkInterval; + boolean argsUpdatedDuringInterval = lastArgsRefreshTs > lastMetricsEvalTs; + return intervalPassed && argsUpdatedDuringInterval; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationArgumentEntry.java new file mode 100644 index 0000000000..10941ebb19 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationArgumentEntry.java @@ -0,0 +1,59 @@ +package org.thingsboard.server.service.cf.ctx.state.aggregation.single; + +import lombok.Data; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; + +import java.util.Map; + +@Data +public class EntityAggregationArgumentEntry implements ArgumentEntry { + + private Map aggIntervals; + + private boolean forceResetPrevious; + + public EntityAggregationArgumentEntry(Map aggIntervals) { + this.aggIntervals = aggIntervals; + } + + @Override + public ArgumentEntryType getType() { + return ArgumentEntryType.ENTITY_AGGREGATION; + } + + @Override + public Object getValue() { + return aggIntervals; + } + + @Override + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof EntityAggregationArgumentEntry entityAggEntry) { + aggIntervals.putAll(entityAggEntry.getAggIntervals()); + } else if (entry instanceof SingleValueArgumentEntry singleValueArgEntry) { + long entryTs = singleValueArgEntry.getTs(); + for (Map.Entry aggIntervalEntry : aggIntervals.entrySet()) { + if (aggIntervalEntry.getKey().belongsToInterval(entryTs)) { + aggIntervalEntry.getValue().setLastArgsRefreshTs(System.currentTimeMillis()); + aggIntervals.put(aggIntervalEntry.getKey(), aggIntervalEntry.getValue()); + return true; + } + } + } + return false; + } + + @Override + public boolean isEmpty() { + return true; + } + + @Override + public TbelCfArg toTbelCfArg() { + return null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java index 6349c635e0..0ae745068e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java @@ -15,15 +15,18 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation.single; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import lombok.Setter; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; import org.thingsboard.server.common.data.cf.configuration.aggregation.single.EntityAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.AggInterval; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -35,20 +38,16 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.HashMap; import java.util.Map; -import static java.util.concurrent.TimeUnit.SECONDS; - public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldState { - private Map aggIntervals = new HashMap<>(); - - @Setter - private long lastArgsRefreshTs = -1; - @Setter - private long lastMetricsEvalTs = -1; + private AggInterval interval; + private long intervalDuration; + private long watermarkDuration; + private long checkInterval; - private long deduplicationIntervalMs = -1; + private Map metrics; - CalculatedFieldProcessingService cfProcessingService; + private CalculatedFieldProcessingService cfProcessingService; public EntityAggregationCalculatedFieldState(EntityId entityId) { super(entityId); @@ -59,7 +58,11 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt super.setCtx(ctx, actorCtx); this.cfProcessingService = ctx.getCfProcessingService(); var configuration = (EntityAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - deduplicationIntervalMs = SECONDS.toMillis(configuration.getDeduplicationIntervalInSec()); + intervalDuration = configuration.getInterval().getIntervalDurationMillis(); + watermarkDuration = configuration.getWatermark().getDuration(); + checkInterval = configuration.getWatermark().getCheckInterval(); + interval = configuration.getInterval(); + metrics = configuration.getMetrics(); } @Override @@ -69,44 +72,141 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception { - long endTs = System.currentTimeMillis(); - long startTs = endTs - 1000; - AggIntervalEntry interval = new AggIntervalEntry(); - - Map metrics = cfProcessingService.fetchArgumentValuesDuringInterval(entityId, interval, ctx); - - Output output = ctx.getOutput(); - lastMetricsEvalTs = System.currentTimeMillis(); - ctx.scheduleReevaluation(deduplicationIntervalMs, actorCtx); - ObjectNode result = toResult(endTs, metrics); - if (result != null) { - return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() - .type(output.getType()) - .scope(output.getScope()) - .result(result) - .build()); + long now = System.currentTimeMillis(); + AggIntervalEntry aggIntervalEntry = new AggIntervalEntry(interval.getCurrentIntervalStartTs(), interval.getCurrentIntervalEndTs()); + boolean exists = false; + for (Map.Entry entry : arguments.entrySet()) { + ArgumentEntry argumentEntry = entry.getValue(); + EntityAggregationArgumentEntry entityAggEntry = (EntityAggregationArgumentEntry) argumentEntry; + Map aggIntervals = entityAggEntry.getAggIntervals(); + exists |= aggIntervals.containsKey(aggIntervalEntry); + } + if (!exists) { + arguments.forEach((argName, argumentEntry) -> { + EntityAggregationArgumentEntry entityAggEntry = (EntityAggregationArgumentEntry) argumentEntry; + entityAggEntry.getAggIntervals().put(aggIntervalEntry, new AggIntervalEntryStatus()); + }); + ctx.scheduleReevaluation(interval.getDelayUntilIntervalEnd(), actorCtx); } - return Futures.immediateFuture(TelemetryCalculatedFieldResult.EMPTY); - } - - protected ObjectNode toResult(long endTs, Map metrics) { - ObjectNode metricsNode = JacksonUtil.newObjectNode(); - for (Map.Entry entry : metrics.entrySet()) { - String metricName = entry.getKey(); + Map> results = new HashMap<>(); + for (Map.Entry entry : arguments.entrySet()) { + String argName = entry.getKey(); ArgumentEntry argumentEntry = entry.getValue(); - if (!argumentEntry.isEmpty()) { - metricsNode.put(metricName, JacksonUtil.toString(argumentEntry.getValue())); + + EntityAggregationArgumentEntry entityAggEntry = (EntityAggregationArgumentEntry) argumentEntry; + Map aggIntervals = entityAggEntry.getAggIntervals(); + for (Map.Entry aggInterval : aggIntervals.entrySet()) { + AggIntervalEntry intervalEntry = aggInterval.getKey(); + AggIntervalEntryStatus entryStatus = aggInterval.getValue(); + + Long startTs = intervalEntry.getStartTs(); + Long endTs = intervalEntry.getEndTs(); + if (now - endTs > watermarkDuration) { + if (entryStatus.getLastArgsRefreshTs() > entryStatus.getLastMetricsEvalTs()) { + String metricName = null; + for (Map.Entry metricEntry : metrics.entrySet()) { + if (((AggKeyInput) metricEntry.getValue().getInput()).getKey().equals(argName)) { + metricName = metricEntry.getKey(); + } + } + ArgumentEntry metric = cfProcessingService.fetchMetricDuringInterval(entityId, intervalEntry, metricName, ctx); + if (!metric.isEmpty()) { + results.computeIfAbsent(intervalEntry, i -> new HashMap<>()).put(argName, metric); + } + } + aggIntervals.remove(intervalEntry); + continue; + } else if (now - startTs >= intervalDuration) { + if (entryStatus.shouldRecalculate(checkInterval)) { + String metricName = null; + for (Map.Entry metricEntry : metrics.entrySet()) { + if (((AggKeyInput) metricEntry.getValue().getInput()).getKey().equals(argName)) { + metricName = metricEntry.getKey(); + } + } + ArgumentEntry metric = cfProcessingService.fetchMetricDuringInterval(entityId, intervalEntry, metricName, ctx); + if (!metric.isEmpty()) { + results.computeIfAbsent(intervalEntry, i -> new HashMap<>()).put(argName, metric); + } + } + } } } - ObjectNode resultNode = JacksonUtil.newObjectNode(); - if (!metricsNode.isEmpty()) { - resultNode.put("ts", endTs); - resultNode.set("values", metricsNode); + ArrayNode result = toResult(results); + if (result.isEmpty()) { + return Futures.immediateFuture(TelemetryCalculatedFieldResult.EMPTY); } - return resultNode; + Output output = ctx.getOutput(); + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .type(output.getType()) + .scope(output.getScope()) + .result(result) + .build()); + +// long now = System.currentTimeMillis(); +// AggIntervalEntry aggIntervalEntry = new AggIntervalEntry(interval.getCurrentIntervalStartTs(), interval.getCurrentIntervalEndTs(), false); +// if (!intervals.containsKey(aggIntervalEntry)) { +// intervals.put(aggIntervalEntry, new AggIntervalEntryStatus()); +// ctx.scheduleReevaluation(interval.getDelayUntilIntervalEnd(), actorCtx); +// } +// ArrayNode results = JacksonUtil.newArrayNode(); +// for (Map.Entry entry : intervals.entrySet()) { +// AggIntervalEntry intervalEntry = entry.getKey(); +// AggIntervalEntryStatus entryStatus = entry.getValue(); +// +// Long startTs = intervalEntry.getStartTs(); +// Long endTs = intervalEntry.getEndTs(); +// if (now - endTs > watermarkDuration) { +// if (entryStatus.getLastArgsRefreshTs() > entryStatus.getLastMetricsEvalTs()) { +// ArgumentEntry metric = cfProcessingService.fetchMetricDuringInterval(entityId, intervalEntry, metricName, ctx); +// ObjectNode result = fetchMetrics(intervalEntry); +// if (result != null) { +// results.add(result); +// } +// } +// intervals.remove(intervalEntry); +// continue; +// } else if (now - startTs >= intervalDuration) { +// if (entryStatus.shouldRecalculate(checkInterval)) { +// ObjectNode result = fetchMetrics(intervalEntry); +// if (result != null) { +// results.add(result); +// } +// } +// } +// } +// if (results.isEmpty()) { +// return Futures.immediateFuture(TelemetryCalculatedFieldResult.EMPTY); +// } +// Output output = ctx.getOutput(); +// return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() +// .type(output.getType()) +// .scope(output.getScope()) +// .result(results) +// .build()); } + protected ArrayNode toResult(Map> results) { + ArrayNode result = JacksonUtil.newArrayNode(); + results.forEach((interval, args) -> { + ObjectNode metricsNode = JacksonUtil.newObjectNode(); + for (Map.Entry entry : args.entrySet()) { + String metricName = entry.getKey(); + ArgumentEntry argumentEntry = entry.getValue(); + if (!argumentEntry.isEmpty()) { + metricsNode.put(metricName, JacksonUtil.toString(argumentEntry.getValue())); + } + } + ObjectNode resultNode = JacksonUtil.newObjectNode(); + if (!metricsNode.isEmpty()) { + resultNode.put("ts", interval.getEndTs()); + resultNode.set("values", metricsNode); + } + result.add(resultNode); + }); + return result; + } @Override public boolean isReady() { diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 121febea7d..8a3e352826 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -204,6 +204,7 @@ public class CalculatedFieldUtils { case ALARM -> new AlarmCalculatedFieldState(id.entityId()); case PROPAGATION -> new PropagationCalculatedFieldState(id.entityId()); case RELATED_ENTITIES_AGGREGATION -> new RelatedEntitiesAggregationCalculatedFieldState(id.entityId()); + case ENTITY_AGGREGATION -> null; // todo }; if (state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 889df54848..88f2f017d3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -529,9 +529,8 @@ actors: configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" # Time in seconds to receive calculation result. calculation_timeout: "${ACTORS_CALCULATION_TIMEOUT_SEC:5}" - alarms: - # Interval in seconds to re-evaluate Alarm rules that have a time schedule. 2 minutes by default. - reevaluation_interval: "${ACTORS_ALARMS_REEVALUATION_INTERVAL_SEC:120}" + # Interval in seconds to re-evaluate calculated fields that have a time schedule. 2 minutes by default. + check_interval: "${ACTORS_CALCULATED_FIELDS_CHECK_INTERVAL_SEC:120}" debug: settings: diff --git a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java index 652e69781d..135b83496f 100644 --- a/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/AlarmRulesTest.java @@ -86,7 +86,7 @@ import static org.testcontainers.shaded.org.awaitility.Awaitility.await; @Slf4j @DaoSqlTest @TestPropertySource(properties = { - "actors.alarms.reevaluation_interval=1" + "actors.calculated_fields.check_interval=1" }) public class AlarmRulesTest extends AbstractControllerTest { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java index 931cb919ec..9d4c7bdaf6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation; -import com.fasterxml.jackson.annotation.JsonIgnore; import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; @@ -57,10 +56,4 @@ public class RelatedEntitiesAggregationCalculatedFieldConfiguration implements A } } - @JsonIgnore - @Override - public boolean requiresScheduledReevaluation() { - return true; - } - } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java index 69f4b30d46..3f5fc7eb96 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.cf.configuration.ArgumentsBasedCalcula import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.AggInterval; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.Watermark; import java.util.Map; @@ -36,8 +37,7 @@ public class EntityAggregationCalculatedFieldConfiguration implements ArgumentsB private Map metrics; private AggInterval interval; - private long deduplicationIntervalInSec; - private long watermark; + private Watermark watermark; private Output output; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java index cf6f825560..966e778240 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java @@ -31,14 +31,19 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = WeekSunSatInterval.class, name = "WEEK_SUN_SAT"), @JsonSubTypes.Type(value = MonthInterval.class, name = "MONTH"), @JsonSubTypes.Type(value = YearInterval.class, name = "YEAR"), - @JsonSubTypes.Type(value = CustomInterval.class, name = "CUSTOM"), - @JsonSubTypes.Type(value = SpecificTimeInterval.class, name = "SPECIFIC_TIME"), + @JsonSubTypes.Type(value = CustomInterval.class, name = "CUSTOM") }) @JsonIgnoreProperties(ignoreUnknown = true) public interface AggInterval { AggIntervalType getType(); + long getIntervalDurationMillis(); + + long getCurrentIntervalStartTs(); + + long getCurrentIntervalEndTs(); + long getDelayUntilIntervalEnd(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java index a5c29b4edf..62185127ed 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java @@ -23,7 +23,6 @@ public enum AggIntervalType { WEEK_SUN_SAT, MONTH, YEAR, - CUSTOM, - SPECIFIC_TIME + CUSTOM } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java index 5790d7545d..b878ef345a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java @@ -15,40 +15,124 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; +import lombok.Data; + import java.time.DayOfWeek; import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; import java.time.ZonedDateTime; import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAdjusters; +@Data public abstract class BaseAggInterval implements AggInterval { protected long offsetMillis; // delay millis since start of interval + @Override + public long getIntervalDurationMillis() { + return getCurrentIntervalEndTs() - getCurrentIntervalStartTs(); + } + + @Override + public long getCurrentIntervalStartTs() { + return getCurrentIntervalStartTs(getType(), 1); + } + + protected long getCurrentIntervalStartTs(AggIntervalType type, int multiplier) { + return getAlignedBoundary(type, multiplier, false).toInstant().toEpochMilli(); + } + + @Override + public long getCurrentIntervalEndTs() { + return getCurrentIntervalEndTs(getType(), 1); + } + + protected long getCurrentIntervalEndTs(AggIntervalType type, int multiplier) { + return getAlignedBoundary(type, multiplier, true).toInstant().toEpochMilli(); + } + @Override public long getDelayUntilIntervalEnd() { return getDelayUntilIntervalEnd(getType(), 1); } - protected long getDelayUntilIntervalEnd(AggIntervalType type, long multiplier) { + protected long getDelayUntilIntervalEnd(AggIntervalType type, int multiplier) { + ZonedDateTime now = ZonedDateTime.now(); + ZonedDateTime currentStart = getAlignedBoundary(type, multiplier, false); + ZonedDateTime nextStart = getAlignedBoundary(type, multiplier, true); + + long periodMillis = Duration.between(currentStart, nextStart).toMillis(); + + // Apply offset: this shifts the grid + long off = offsetMillis % periodMillis; + if (off < 0) off += periodMillis; + + // Compute the offset-aligned start times + ZonedDateTime offsetCurrentStart = currentStart.plus(Duration.ofMillis(off)); + ZonedDateTime offsetNextStart = offsetCurrentStart.plus(Duration.ofMillis(periodMillis)); + + // If we are already past the current offset start, move to the next + ZonedDateTime target = offsetCurrentStart.isAfter(now) ? offsetCurrentStart : offsetNextStart; + // todo fix + return Math.max(Duration.between(now, target).toMillis(), 0); + } + + protected ZonedDateTime getAlignedBoundary(AggIntervalType type, int multiplier, boolean next) { ZonedDateTime now = ZonedDateTime.now(); - ZonedDateTime next; - - switch (getType()) { - case HOUR -> next = now.plusHours(multiplier).truncatedTo(ChronoUnit.HOURS); - case DAY -> next = now.plusDays(multiplier).truncatedTo(ChronoUnit.DAYS); - case WEEK -> next = now.plusWeeks(multiplier).with(DayOfWeek.MONDAY).truncatedTo(ChronoUnit.DAYS); - case WEEK_SUN_SAT -> next = now.plusWeeks(multiplier).with(DayOfWeek.SUNDAY).truncatedTo(ChronoUnit.DAYS); - case MONTH -> next = now.plusMonths(multiplier).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS); - case YEAR -> next = now.plusYears(multiplier).withDayOfYear(1).truncatedTo(ChronoUnit.DAYS); - default -> throw new IllegalArgumentException("Unsupported type: " + getType()); - } - - long delayMillis = Duration.between(now, next).toMillis(); - if (offsetMillis > 0) { - delayMillis += offsetMillis; - } - - return Math.max(delayMillis, 0); + + return switch (type) { + case HOUR -> alignByHour(now, multiplier, next); + case DAY -> alignByDay(now, multiplier, next); + case WEEK -> alignByWeek(now, multiplier, DayOfWeek.MONDAY, next); + case WEEK_SUN_SAT -> alignByWeek(now, multiplier, DayOfWeek.SUNDAY, next); + case MONTH -> alignByMonth(now, multiplier, next); + case YEAR -> alignByYear(now, multiplier, next); + default -> throw new IllegalArgumentException("Unsupported type: " + type); + }; + } + + private ZonedDateTime alignByHour(ZonedDateTime now, int multiplier, boolean next) { + ZonedDateTime startOfDay = now.truncatedTo(ChronoUnit.DAYS); + long hoursSinceMidnight = Duration.between(startOfDay, now).toHours(); + long aligned = (hoursSinceMidnight / multiplier) * multiplier; + if (next) aligned += multiplier; + return startOfDay.plusHours(aligned); + } + + private ZonedDateTime alignByDay(ZonedDateTime now, int multiplier, boolean next) { + long daysSinceEpoch = now.toLocalDate().toEpochDay(); + long aligned = (daysSinceEpoch / multiplier) * multiplier; + if (next) aligned += multiplier; + long diff = aligned - daysSinceEpoch; + return now.truncatedTo(ChronoUnit.DAYS).plusDays(diff); + } + + private ZonedDateTime alignByWeek(ZonedDateTime now, int multiplier, DayOfWeek startOfWeekDay, boolean next) { + ZonedDateTime startOfWeek = now.with(TemporalAdjusters.previousOrSame(startOfWeekDay)) + .truncatedTo(ChronoUnit.DAYS); + long weeksSinceEpoch = ChronoUnit.WEEKS.between( + ZonedDateTime.ofInstant(Instant.EPOCH, now.getZone()), startOfWeek); + long aligned = (weeksSinceEpoch / multiplier) * multiplier; + if (next) aligned += multiplier; + return startOfWeek.plusWeeks(aligned - weeksSinceEpoch); + } + + private ZonedDateTime alignByMonth(ZonedDateTime now, int multiplier, boolean next) { + ZonedDateTime startOfMonth = now.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS); + long monthsSinceEpoch = now.getYear() * 12L + now.getMonthValue() - 1; + long aligned = (monthsSinceEpoch / multiplier) * multiplier; + if (next) aligned += multiplier; + return startOfMonth.plusMonths(aligned - monthsSinceEpoch); + } + + private ZonedDateTime alignByYear(ZonedDateTime now, int multiplier, boolean next) { + int year = now.getYear(); + int aligned = (year / multiplier) * multiplier; + if (next) aligned += multiplier; + return ZonedDateTime.of(LocalDate.of(aligned, 1, 1), LocalTime.MIDNIGHT, now.getZone()); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java index 585e11d955..d875794a18 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java @@ -25,6 +25,21 @@ public class CustomInterval extends BaseAggInterval { return AggIntervalType.CUSTOM; } + @Override + public long getIntervalDurationMillis() { + return getCurrentIntervalEndTs() - getCurrentIntervalStartTs(); + } + + @Override + public long getCurrentIntervalStartTs() { + return super.getCurrentIntervalStartTs(internalIntervalType, multiplier); + } + + @Override + public long getCurrentIntervalEndTs() { + return super.getCurrentIntervalEndTs(internalIntervalType, multiplier); + } + @Override public long getDelayUntilIntervalEnd() { return super.getDelayUntilIntervalEnd(internalIntervalType, multiplier); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java index 19620c6fda..01cbdf2e97 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; +import lombok.Data; + +@Data public class DayInterval extends BaseAggInterval { @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java index 303ff8305e..ce84b57ae1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; +import lombok.Data; + +@Data public class HourInterval extends BaseAggInterval { @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java index 1d7eaf14bc..fe8d60f41c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; +import lombok.Data; + +@Data public class MonthInterval extends BaseAggInterval { @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/SpecificTimeInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/SpecificTimeInterval.java deleted file mode 100644 index 2b44366f8b..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/SpecificTimeInterval.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; - -import java.time.LocalTime; - -public class SpecificTimeInterval implements AggInterval { - - public Long startMillis; // start millis since start of day - public Long endMillis; // end millis since start of day - - @Override - public AggIntervalType getType() { - return AggIntervalType.SPECIFIC_TIME; - } - - @Override - public long getDelayUntilIntervalEnd() { - long nowMillis = LocalTime.now().toNanoOfDay() / 1_000_000L; - long delayMillis; - if (nowMillis < endMillis) { - delayMillis = endMillis - nowMillis; // later today - } else { - delayMillis = (24 * 60 * 60 * 1000L - nowMillis) + endMillis; // next day - } - return delayMillis; - } - -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/Watermark.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/Watermark.java new file mode 100644 index 0000000000..44dd6404f6 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/Watermark.java @@ -0,0 +1,11 @@ +package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; + +import lombok.Data; + +@Data +public class Watermark { + + private long duration; + private long checkInterval; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java index 4dd10a2747..5a93076772 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; +import lombok.Data; + +@Data public class WeekInterval extends BaseAggInterval { @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java index c12e7dc584..c70dd79a9f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; +import lombok.Data; + +@Data public class WeekSunSatInterval extends BaseAggInterval { @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java index 24c6c72ed1..3c600064d1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; +import lombok.Data; + +@Data public class YearInterval extends BaseAggInterval { @Override From a21bb92e50d26276ac06ff16c0326cd7f12d5bf3 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 30 Oct 2025 12:19:51 +0200 Subject: [PATCH 03/40] added state proto --- ...CalculatedFieldEntityMessageProcessor.java | 11 +- ...alculatedFieldManagerMessageProcessor.java | 17 +- ...tractCalculatedFieldProcessingService.java | 41 +--- .../service/cf/CalculatedFieldCache.java | 4 +- .../cf/CalculatedFieldProcessingService.java | 2 - .../cf/DefaultCalculatedFieldCache.java | 11 +- ...faultCalculatedFieldProcessingService.java | 11 +- .../DefaultCalculatedFieldQueueService.java | 9 +- .../cf/ctx/state/CalculatedFieldCtx.java | 6 - .../single/AggIntervalEntryStatus.java | 26 +- .../EntityAggregationArgumentEntry.java | 15 ++ ...EntityAggregationCalculatedFieldState.java | 226 ++++++++++-------- .../server/utils/CalculatedFieldUtils.java | 45 +++- ...gregationCalculatedFieldConfiguration.java | 2 - .../single/interval/AggInterval.java | 4 +- .../single/interval/AggIntervalType.java | 4 +- .../single/interval/BaseAggInterval.java | 72 +++--- .../single/interval/CustomInterval.java | 5 +- .../single/interval/MinInterval.java | 28 +++ .../single/interval/Watermark.java | 15 ++ common/proto/src/main/proto/queue.proto | 9 + 21 files changed, 346 insertions(+), 217 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MinInterval.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 083af60945..69ea705c60 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -31,7 +31,6 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; -import org.thingsboard.server.common.data.cf.configuration.aggregation.single.EntityAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -54,6 +53,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.EntityAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -127,6 +127,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { relatedEntitiesAggState.scheduleReevaluation(); } + if (state instanceof EntityAggregationCalculatedFieldState entityAggState) { + entityAggState.scheduleReevaluation(); + } states.put(cfId, state); } else { removeState(cfId); @@ -441,11 +444,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM GeofencingCalculatedFieldState geofencingState = (GeofencingCalculatedFieldState) state; geofencingState.updateLastDynamicArgumentsRefreshTs(); } - if (ctx.getCfType() == CalculatedFieldType.ENTITY_AGGREGATION) { - var configuration = (EntityAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - long delayUntilIntervalEnd = configuration.getInterval().getDelayUntilIntervalEnd(); - ctx.scheduleReevaluation(delayUntilIntervalEnd, actorCtx); - } + Map arguments = fetchArguments(ctx); state.update(arguments, ctx); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 075918e6e8..00e1dd3f96 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -34,7 +34,6 @@ import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.aggregation.single.EntityAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -94,7 +93,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final Map> entityIdCalculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFieldLinks = new HashMap<>(); private final Map> ownerEntities = new HashMap<>(); - private final Map entityAggCalculatedFields = new HashMap<>(); private ScheduledFuture cfsReevaluationTask; private final CalculatedFieldProcessingService cfExecService; @@ -307,8 +305,10 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onRelationChangedEvent(ComponentLifecycleMsg msg, TbCallback callback) { Function> relationAction = switch (msg.getEvent()) { - case RELATION_UPDATED -> relatedId -> (entityId, ctx, cb) -> initRelatedEntity(entityId, relatedId, ctx, cb); - case RELATION_DELETED -> relatedId -> (entityId, ctx, cb) -> deleteRelatedEntity(entityId, relatedId, ctx, cb); + case RELATION_UPDATED -> + relatedId -> (entityId, ctx, cb) -> initRelatedEntity(entityId, relatedId, ctx, cb); + case RELATION_DELETED -> + relatedId -> (entityId, ctx, cb) -> deleteRelatedEntity(entityId, relatedId, ctx, cb); default -> null; }; @@ -372,9 +372,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } calculatedFields.put(cf.getId(), cfCtx); - if (cf.getConfiguration() instanceof EntityAggregationCalculatedFieldConfiguration aggConfig) { - entityAggCalculatedFields.put(cf.getId(), cfCtx); - } // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); @@ -406,9 +403,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(newCfCtx).eventEntity(newCfCtx.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(newCf.getId(), newCfCtx); - if (newCf.getConfiguration() instanceof EntityAggregationCalculatedFieldConfiguration aggConfig) { - entityAggCalculatedFields.put(newCf.getId(), newCfCtx); - } List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new CopyOnWriteArrayList<>(); boolean found = false; @@ -775,9 +769,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(cf.getId(), cfCtx); - if (cf.getConfiguration() instanceof EntityAggregationCalculatedFieldConfiguration aggConfig) { - entityAggCalculatedFields.put(cf.getId(), cfCtx); - } // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 2110a78f5c..48bce35b3e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -40,7 +40,6 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -62,7 +61,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -307,42 +305,6 @@ public abstract class AbstractCalculatedFieldProcessingService { }; } - protected Map fetchMetricsDuringInterval(EntityId entityId, AggIntervalEntry interval, CalculatedFieldCtx ctx) throws Exception { - var config = (EntityAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - Map metricsResult = new HashMap<>(); - - for (Entry entry : config.getMetrics().entrySet()) { - String metricName = entry.getKey(); - AggMetric metric = entry.getValue(); - AggFunction function = metric.getFunction(); - - AggKeyInput input = (AggKeyInput) metric.getInput(); - String argName = input.getKey(); - Argument argument = ctx.getArguments().get(argName); - String key = argument.getRefEntityKey().getKey(); - - BaseReadTsKvQuery query = new BaseReadTsKvQuery(key, interval.getStartTs(), interval.getEndTs(), 0, 1, Aggregation.valueOf(function.name())); - log.trace("[{}][{}] Fetching timeseries for query {}", ctx.getTenantId(), entityId, query); - ListenableFuture> tsFuture = timeseriesService.findAll(ctx.getTenantId(), entityId, List.of(query)); - ListenableFuture argumentEntryFut = Futures.transform(tsFuture, timeSeries -> { - log.debug("[{}][{}] Fetched {} timeseries for query {}", ctx.getTenantId(), entityId, timeSeries == null ? 0 : timeSeries.size(), query); - if (timeSeries == null || timeSeries.isEmpty()) { - return new SingleValueArgumentEntry(); - } - return ArgumentEntry.createSingleValueArgument(timeSeries.get(0)); - }, calculatedFieldCallbackExecutor); - - // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. - // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. - // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, - // but this will significantly complicate the code. - ArgumentEntry argumentEntry = argumentEntryFut.get(1, TimeUnit.MINUTES); - metricsResult.put(metricName, argumentEntry); - } - - return metricsResult; - } - protected ArgumentEntry fetchMetricDuringInterval(EntityId entityId, AggIntervalEntry interval, String metricName, CalculatedFieldCtx ctx) throws Exception { var config = (EntityAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); @@ -354,7 +316,8 @@ public abstract class AbstractCalculatedFieldProcessingService { Argument argument = ctx.getArguments().get(argName); String key = argument.getRefEntityKey().getKey(); - BaseReadTsKvQuery query = new BaseReadTsKvQuery(key, interval.getStartTs(), interval.getEndTs(), 0, 1, Aggregation.valueOf(function.name())); + long intervalMs = interval.getEndTs() - interval.getStartTs(); + BaseReadTsKvQuery query = new BaseReadTsKvQuery(key, interval.getStartTs(), interval.getEndTs(), intervalMs, 1, Aggregation.valueOf(function.name())); log.trace("[{}][{}] Fetching timeseries for query {}", ctx.getTenantId(), entityId, query); ListenableFuture> tsFuture = timeseriesService.findAll(ctx.getTenantId(), entityId, List.of(query)); ListenableFuture argumentEntryFut = Futures.transform(tsFuture, timeSeries -> { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index 27e989de70..5842363229 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -38,7 +38,9 @@ public interface CalculatedFieldCache { List getCalculatedFieldCtxsByEntityId(EntityId entityId); - List getAggCalculatedFieldCtxsByFilter(Predicate relatedEntityFilter); + List getRelatedEntitiesAggCalculatedFieldCtxsByFilter(Predicate relatedEntityFilter); + + List getEntityAggCalculatedFieldCtxsByFilter(Predicate entityAggCfFilter); boolean hasCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index 54796e18eb..e65df09f98 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -38,8 +38,6 @@ public interface CalculatedFieldProcessingService { Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments); - Map fetchMetricsDuringInterval(EntityId entityId, AggIntervalEntry interval, CalculatedFieldCtx ctx) throws Exception; - ArgumentEntry fetchMetricDuringInterval(EntityId entityId, AggIntervalEntry interval, String argName, CalculatedFieldCtx ctx) throws Exception; void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult result, List cfIds, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index e802bd7677..4f2b7a459c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -148,7 +148,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } @Override - public List getAggCalculatedFieldCtxsByFilter(Predicate relatedEntityFilter) { + public List getRelatedEntitiesAggCalculatedFieldCtxsByFilter(Predicate relatedEntityFilter) { return calculatedFields.values().stream() .filter(cf -> CalculatedFieldType.RELATED_ENTITIES_AGGREGATION.equals(cf.getType())) .map(cf -> getCalculatedFieldCtx(cf.getId())) @@ -156,6 +156,15 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { .toList(); } + @Override + public List getEntityAggCalculatedFieldCtxsByFilter(Predicate entityAggCfFilter) { + return calculatedFields.values().stream() + .filter(cf -> CalculatedFieldType.ENTITY_AGGREGATION.equals(cf.getType())) + .map(cf -> getCalculatedFieldCtx(cf.getId())) + .filter(entityAggCfFilter) + .toList(); + } + @Override public boolean hasCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter) { List entityCfs = getCalculatedFieldCtxsByEntityId(entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index 35f767fde2..c6a66d100e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -92,8 +92,10 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF @Override public Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId) { return switch (ctx.getCfType()) { - case GEOFENCING -> resolveArgumentFutures(fetchGeofencingCalculatedFieldArguments(ctx, entityId, true, System.currentTimeMillis())); - case PROPAGATION -> resolveArgumentFutures(Map.of(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId))); + case GEOFENCING -> + resolveArgumentFutures(fetchGeofencingCalculatedFieldArguments(ctx, entityId, true, System.currentTimeMillis())); + case PROPAGATION -> + resolveArgumentFutures(Map.of(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId))); default -> Collections.emptyMap(); }; } @@ -112,11 +114,6 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF return resolveArgumentFutures(argFutures); } - @Override - public Map fetchMetricsDuringInterval(EntityId entityId, AggIntervalEntry interval, CalculatedFieldCtx ctx) throws Exception { - return super.fetchMetricsDuringInterval(entityId, interval, ctx); - } - @Override public ArgumentEntry fetchMetricDuringInterval(EntityId entityId, AggIntervalEntry interval, String metricName, CalculatedFieldCtx ctx) throws Exception { return super.fetchMetricDuringInterval(entityId, interval, metricName, ctx); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index c7e369a862..09c6fc9c38 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -188,8 +188,13 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } } - List cfCtxs = calculatedFieldCache.getAggCalculatedFieldCtxsByFilter(relatedEntityFilter); - for (CalculatedFieldCtx cfCtx : cfCtxs) { + List entityAggCfCtxs = calculatedFieldCache.getEntityAggCalculatedFieldCtxsByFilter(filter); + if (!entityAggCfCtxs.isEmpty()) { + return true; + } + + List relatedEntityAggCfCtxs = calculatedFieldCache.getRelatedEntitiesAggCalculatedFieldCtxsByFilter(relatedEntityFilter); + for (CalculatedFieldCtx cfCtx : relatedEntityAggCfCtxs) { if (cfCtx.getCalculatedField().getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig) { RelationPathLevel relation = aggConfig.getRelation(); EntitySearchDirection inverseDirection = switch (relation.direction()) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 0405687b4c..c0208825a8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -37,7 +37,6 @@ import org.thingsboard.server.common.data.cf.configuration.AlarmCalculatedFieldC import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ArgumentsBasedCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ExpressionBasedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration; @@ -46,8 +45,6 @@ import org.thingsboard.server.common.data.cf.configuration.ScheduledUpdateSuppor import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.aggregation.single.EntityAggregationCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.AggInterval; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -59,7 +56,6 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; @@ -98,8 +94,6 @@ public class CalculatedFieldCtx implements Closeable { private String expression; private boolean useLatestTs; private boolean requiresScheduledReevaluation; -// -// private long lastReevaluationTs; private ActorSystemContext systemContext; private TbelInvokeService tbelInvokeService; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntryStatus.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntryStatus.java index 09fa341961..fa9bbd5a54 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntryStatus.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntryStatus.java @@ -1,11 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.cf.ctx.state.aggregation.single; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.Setter; @Data @NoArgsConstructor +@AllArgsConstructor public class AggIntervalEntryStatus { @Setter @@ -19,8 +36,13 @@ public class AggIntervalEntryStatus { public boolean shouldRecalculate(long checkInterval) { boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - checkInterval; - boolean argsUpdatedDuringInterval = lastArgsRefreshTs > lastMetricsEvalTs; - return intervalPassed && argsUpdatedDuringInterval; + boolean argsUpdatedDuringInterval = lastArgsRefreshTs > -1; + if (intervalPassed && argsUpdatedDuringInterval) { + lastMetricsEvalTs = System.currentTimeMillis(); + lastArgsRefreshTs = -1; + return true; + } + return false; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationArgumentEntry.java index 10941ebb19..c1d1b5d807 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationArgumentEntry.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.cf.ctx.state.aggregation.single; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java index 0ae745068e..33a8003e03 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java @@ -35,6 +35,7 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.BaseCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import java.util.Comparator; import java.util.HashMap; import java.util.Map; @@ -44,15 +45,54 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt private long intervalDuration; private long watermarkDuration; private long checkInterval; - private Map metrics; + private final Map> intervals = new HashMap<>(); + private CalculatedFieldProcessingService cfProcessingService; public EntityAggregationCalculatedFieldState(EntityId entityId) { super(entityId); } + public void scheduleReevaluation() { + fillMissingIntervals(interval.getCurrentIntervalEndTs(), intervalDuration); + prepareIntervals(); + long now = System.currentTimeMillis(); + intervals.forEach((intervalEntry, argumentIntervalStatuses) -> { + if (intervalEntry.belongsToInterval(now)) { + ctx.scheduleReevaluation(interval.getDelayUntilIntervalEnd(), actorCtx); + } else { + if (intervalEntry.getEndTs() <= now) { + ctx.scheduleReevaluation(checkInterval, actorCtx); + } + } + }); + } + + private void fillMissingIntervals(long currentIntervalEndTs, long intervalDuration) { + AggIntervalEntry lastIntervalEntry = intervals.keySet().stream().max(Comparator.comparing(AggIntervalEntry::getEndTs)).orElse(null); + if (lastIntervalEntry == null) { + return; + } + + long nextStartTs = lastIntervalEntry.getEndTs(); + long nextEndTs = nextStartTs + intervalDuration; + + while (nextEndTs <= currentIntervalEndTs) { + AggIntervalEntry missingAggIntervalEntry = new AggIntervalEntry(nextStartTs, nextEndTs); + + arguments.forEach((argName, argumentEntry) -> { + var entityAggEntry = (EntityAggregationArgumentEntry) argumentEntry; + entityAggEntry.getAggIntervals().put(missingAggIntervalEntry, new AggIntervalEntryStatus()); + intervals.computeIfAbsent(missingAggIntervalEntry, i -> new HashMap<>()).put(argName, new AggIntervalEntryStatus()); + }); + + nextStartTs = nextEndTs; + nextEndTs += intervalDuration; + } + } + @Override public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) { super.setCtx(ctx, actorCtx); @@ -72,67 +112,17 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception { + createIntervalIfNotExist(); + prepareIntervals(); long now = System.currentTimeMillis(); - AggIntervalEntry aggIntervalEntry = new AggIntervalEntry(interval.getCurrentIntervalStartTs(), interval.getCurrentIntervalEndTs()); - boolean exists = false; - for (Map.Entry entry : arguments.entrySet()) { - ArgumentEntry argumentEntry = entry.getValue(); - EntityAggregationArgumentEntry entityAggEntry = (EntityAggregationArgumentEntry) argumentEntry; - Map aggIntervals = entityAggEntry.getAggIntervals(); - exists |= aggIntervals.containsKey(aggIntervalEntry); - } - if (!exists) { - arguments.forEach((argName, argumentEntry) -> { - EntityAggregationArgumentEntry entityAggEntry = (EntityAggregationArgumentEntry) argumentEntry; - entityAggEntry.getAggIntervals().put(aggIntervalEntry, new AggIntervalEntryStatus()); - }); - ctx.scheduleReevaluation(interval.getDelayUntilIntervalEnd(), actorCtx); - } Map> results = new HashMap<>(); - for (Map.Entry entry : arguments.entrySet()) { - String argName = entry.getKey(); - ArgumentEntry argumentEntry = entry.getValue(); - - EntityAggregationArgumentEntry entityAggEntry = (EntityAggregationArgumentEntry) argumentEntry; - Map aggIntervals = entityAggEntry.getAggIntervals(); - for (Map.Entry aggInterval : aggIntervals.entrySet()) { - AggIntervalEntry intervalEntry = aggInterval.getKey(); - AggIntervalEntryStatus entryStatus = aggInterval.getValue(); - - Long startTs = intervalEntry.getStartTs(); - Long endTs = intervalEntry.getEndTs(); - if (now - endTs > watermarkDuration) { - if (entryStatus.getLastArgsRefreshTs() > entryStatus.getLastMetricsEvalTs()) { - String metricName = null; - for (Map.Entry metricEntry : metrics.entrySet()) { - if (((AggKeyInput) metricEntry.getValue().getInput()).getKey().equals(argName)) { - metricName = metricEntry.getKey(); - } - } - ArgumentEntry metric = cfProcessingService.fetchMetricDuringInterval(entityId, intervalEntry, metricName, ctx); - if (!metric.isEmpty()) { - results.computeIfAbsent(intervalEntry, i -> new HashMap<>()).put(argName, metric); - } - } - aggIntervals.remove(intervalEntry); - continue; - } else if (now - startTs >= intervalDuration) { - if (entryStatus.shouldRecalculate(checkInterval)) { - String metricName = null; - for (Map.Entry metricEntry : metrics.entrySet()) { - if (((AggKeyInput) metricEntry.getValue().getInput()).getKey().equals(argName)) { - metricName = metricEntry.getKey(); - } - } - ArgumentEntry metric = cfProcessingService.fetchMetricDuringInterval(entityId, intervalEntry, metricName, ctx); - if (!metric.isEmpty()) { - results.computeIfAbsent(intervalEntry, i -> new HashMap<>()).put(argName, metric); - } - } - } - } + for (Map.Entry> entry : intervals.entrySet()) { + AggIntervalEntry intervalEntry = entry.getKey(); + Map args = entry.getValue(); + processInterval(now, intervalEntry, args, results); } + ArrayNode result = toResult(results); if (result.isEmpty()) { return Futures.immediateFuture(TelemetryCalculatedFieldResult.EMPTY); @@ -143,48 +133,86 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt .scope(output.getScope()) .result(result) .build()); + } + + private void prepareIntervals() { + arguments.forEach((argName, entry) -> { + var argEntry = (EntityAggregationArgumentEntry) entry; + argEntry.getAggIntervals().forEach((intervalEntry, status) -> + intervals.computeIfAbsent(intervalEntry, i -> new HashMap<>()).put(argName, status) + ); + }); + } + + private void createIntervalIfNotExist() { + AggIntervalEntry currentInterval = new AggIntervalEntry(interval.getCurrentIntervalStartTs(), interval.getCurrentIntervalEndTs()); + if (intervals.containsKey(currentInterval)) { + return; + } + arguments.forEach((argName, argumentEntry) -> { + var entityAggEntry = (EntityAggregationArgumentEntry) argumentEntry; + entityAggEntry.getAggIntervals().put(currentInterval, new AggIntervalEntryStatus()); + intervals.computeIfAbsent(currentInterval, i -> new HashMap<>()).put(argName, new AggIntervalEntryStatus()); + }); + ctx.scheduleReevaluation(interval.getDelayUntilIntervalEnd(), actorCtx); + } + + private void processInterval(long now, AggIntervalEntry intervalEntry, Map args, + Map> results) throws Exception { + long startTs = intervalEntry.getStartTs(); + long endTs = intervalEntry.getEndTs(); + + if (now - endTs > watermarkDuration) { + handleExpiredInterval(intervalEntry, args, results); + intervals.remove(intervalEntry); + } else if (now - startTs >= intervalDuration) { + handleActiveInterval(intervalEntry, args, results); + } + } + + private void handleExpiredInterval(AggIntervalEntry intervalEntry, + Map args, + Map> results) throws Exception { + for (Map.Entry argStatus : args.entrySet()) { + String argName = argStatus.getKey(); + AggIntervalEntryStatus argEntryIntervalStatus = argStatus.getValue(); + if (argEntryIntervalStatus.getLastArgsRefreshTs() > argEntryIntervalStatus.getLastMetricsEvalTs()) { + processMetric(intervalEntry, argName, results); + } + } + } + + private void handleActiveInterval(AggIntervalEntry intervalEntry, + Map args, + Map> results) throws Exception { + for (Map.Entry argStatus : args.entrySet()) { + String argName = argStatus.getKey(); + AggIntervalEntryStatus argEntryIntervalStatus = argStatus.getValue(); + if (argEntryIntervalStatus.shouldRecalculate(checkInterval)) { + processMetric(intervalEntry, argName, results); + ctx.scheduleReevaluation(checkInterval, actorCtx); + } + } + } + + private void processMetric(AggIntervalEntry intervalEntry, + String argName, + Map> results) throws Exception { + String metricName = findMetricName(argName); + if (metricName != null) { + ArgumentEntry metric = cfProcessingService.fetchMetricDuringInterval(entityId, intervalEntry, metricName, ctx); + if (!metric.isEmpty()) { + results.computeIfAbsent(intervalEntry, i -> new HashMap<>()).put(metricName, metric); + } + } + } -// long now = System.currentTimeMillis(); -// AggIntervalEntry aggIntervalEntry = new AggIntervalEntry(interval.getCurrentIntervalStartTs(), interval.getCurrentIntervalEndTs(), false); -// if (!intervals.containsKey(aggIntervalEntry)) { -// intervals.put(aggIntervalEntry, new AggIntervalEntryStatus()); -// ctx.scheduleReevaluation(interval.getDelayUntilIntervalEnd(), actorCtx); -// } -// ArrayNode results = JacksonUtil.newArrayNode(); -// for (Map.Entry entry : intervals.entrySet()) { -// AggIntervalEntry intervalEntry = entry.getKey(); -// AggIntervalEntryStatus entryStatus = entry.getValue(); -// -// Long startTs = intervalEntry.getStartTs(); -// Long endTs = intervalEntry.getEndTs(); -// if (now - endTs > watermarkDuration) { -// if (entryStatus.getLastArgsRefreshTs() > entryStatus.getLastMetricsEvalTs()) { -// ArgumentEntry metric = cfProcessingService.fetchMetricDuringInterval(entityId, intervalEntry, metricName, ctx); -// ObjectNode result = fetchMetrics(intervalEntry); -// if (result != null) { -// results.add(result); -// } -// } -// intervals.remove(intervalEntry); -// continue; -// } else if (now - startTs >= intervalDuration) { -// if (entryStatus.shouldRecalculate(checkInterval)) { -// ObjectNode result = fetchMetrics(intervalEntry); -// if (result != null) { -// results.add(result); -// } -// } -// } -// } -// if (results.isEmpty()) { -// return Futures.immediateFuture(TelemetryCalculatedFieldResult.EMPTY); -// } -// Output output = ctx.getOutput(); -// return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() -// .type(output.getType()) -// .scope(output.getScope()) -// .result(results) -// .build()); + private String findMetricName(String argName) { + return metrics.entrySet().stream() + .filter(e -> ((AggKeyInput) e.getValue().getInput()).getKey().equals(argName)) + .map(Map.Entry::getKey) + .findFirst() + .orElse(null); } protected ArrayNode toResult(Map> results) { diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 8a3e352826..780df5b795 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.util.KvProtoUtil; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.gen.transport.TransportProtos.AlarmRuleStateProto; import org.thingsboard.server.gen.transport.TransportProtos.AlarmStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ArgumentIntervalProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; @@ -47,6 +48,10 @@ import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntryStatus; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.EntityAggregationArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.EntityAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; @@ -97,14 +102,21 @@ public class CalculatedFieldUtils { state.getArguments().forEach((argName, argEntry) -> { switch (argEntry.getType()) { - case SINGLE_VALUE -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); - case TS_ROLLING -> builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry)); - case GEOFENCING -> builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); + case SINGLE_VALUE -> + builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); + case TS_ROLLING -> + builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry)); + case GEOFENCING -> + builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); case RELATED_ENTITIES -> { RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry; relatedEntitiesArgumentEntry.getEntityInputs() .forEach((entityId, entry) -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) entry))); } + case ENTITY_AGGREGATION -> { + EntityAggregationArgumentEntry entityAggregationArgumentEntry = (EntityAggregationArgumentEntry) argEntry; + entityAggregationArgumentEntry.getAggIntervals().forEach((interval, argumentStatus) -> builder.addAggregationArguments(toArgumentIntervalProto(argName, interval, argumentStatus))); + } } }); if (state instanceof AlarmCalculatedFieldState alarmState) { @@ -158,6 +170,16 @@ public class CalculatedFieldUtils { return builder.build(); } + public static ArgumentIntervalProto toArgumentIntervalProto(String argName, AggIntervalEntry intervalEntry, AggIntervalEntryStatus argumentStatus) { + return ArgumentIntervalProto.newBuilder() + .setArgName(argName) + .setStartTs(intervalEntry.getStartTs()) + .setEndTs(intervalEntry.getEndTs()) + .setLastArgsRefreshTs(argumentStatus.getLastArgsRefreshTs()) + .setLastMetricsEvalTs(argumentStatus.getLastMetricsEvalTs()) + .build(); + } + public static TsRollingArgumentProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { TsRollingArgumentProto.Builder builder = TsRollingArgumentProto.newBuilder() .setKey(argName) @@ -204,7 +226,7 @@ public class CalculatedFieldUtils { case ALARM -> new AlarmCalculatedFieldState(id.entityId()); case PROPAGATION -> new PropagationCalculatedFieldState(id.entityId()); case RELATED_ENTITIES_AGGREGATION -> new RelatedEntitiesAggregationCalculatedFieldState(id.entityId()); - case ENTITY_AGGREGATION -> null; // todo + case ENTITY_AGGREGATION -> new EntityAggregationCalculatedFieldState(id.entityId()); }; if (state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { @@ -222,6 +244,21 @@ public class CalculatedFieldUtils { return relatedEntitiesAggState; } + if (state instanceof EntityAggregationCalculatedFieldState entityAggregationState) { + Map arguments = new HashMap<>(); + + proto.getAggregationArgumentsList().forEach(argProto -> { + AggIntervalEntry intervalEntry = new AggIntervalEntry(argProto.getStartTs(), argProto.getEndTs()); + AggIntervalEntryStatus intervalStatus = new AggIntervalEntryStatus(argProto.getLastArgsRefreshTs(), argProto.getLastMetricsEvalTs()); + EntityAggregationArgumentEntry argEntry = arguments.computeIfAbsent(argProto.getArgName(), name -> new EntityAggregationArgumentEntry(new HashMap<>())); + argEntry.getAggIntervals().put(intervalEntry, intervalStatus); + }); + + entityAggregationState.getArguments().putAll(arguments); + + return entityAggregationState; + } + proto.getSingleValueArgumentsList().forEach(argProto -> state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java index 3f5fc7eb96..15b85af7bc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java @@ -35,10 +35,8 @@ public class EntityAggregationCalculatedFieldConfiguration implements ArgumentsB @Valid @NotEmpty private Map metrics; - private AggInterval interval; private Watermark watermark; - private Output output; @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java index 966e778240..4c21c80086 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java @@ -31,7 +31,9 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = WeekSunSatInterval.class, name = "WEEK_SUN_SAT"), @JsonSubTypes.Type(value = MonthInterval.class, name = "MONTH"), @JsonSubTypes.Type(value = YearInterval.class, name = "YEAR"), - @JsonSubTypes.Type(value = CustomInterval.class, name = "CUSTOM") + @JsonSubTypes.Type(value = CustomInterval.class, name = "CUSTOM"), + + @JsonSubTypes.Type(value = MinInterval.class, name = "MIN")// todo: delete. used only for tests }) @JsonIgnoreProperties(ignoreUnknown = true) public interface AggInterval { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java index 62185127ed..82cf46c6ff 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java @@ -23,6 +23,8 @@ public enum AggIntervalType { WEEK_SUN_SAT, MONTH, YEAR, - CUSTOM + CUSTOM, + + MIN// todo: delete } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java index b878ef345a..cc207133fa 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java @@ -33,7 +33,19 @@ public abstract class BaseAggInterval implements AggInterval { @Override public long getIntervalDurationMillis() { - return getCurrentIntervalEndTs() - getCurrentIntervalStartTs(); + return getIntervalDurationMillis(getType(), 1); + } + + public long getIntervalDurationMillis(AggIntervalType type, int multiplier) { + return switch (type) { + case MIN -> Duration.ofMinutes(multiplier).toMillis(); + case HOUR -> Duration.ofHours(multiplier).toMillis(); + case DAY -> Duration.ofDays(multiplier).toMillis(); + case WEEK, WEEK_SUN_SAT -> Duration.ofDays(7L * multiplier).toMillis(); + case MONTH -> Duration.ofDays(Math.round(30 * multiplier)).toMillis(); // average + case YEAR -> Duration.ofDays(Math.round(365 * multiplier)).toMillis(); + default -> throw new IllegalArgumentException("Unsupported type: " + type); + }; } @Override @@ -42,7 +54,11 @@ public abstract class BaseAggInterval implements AggInterval { } protected long getCurrentIntervalStartTs(AggIntervalType type, int multiplier) { - return getAlignedBoundary(type, multiplier, false).toInstant().toEpochMilli(); + ZonedDateTime now = ZonedDateTime.now(); + ZonedDateTime shiftedNow = now.minus(Duration.ofMillis(offsetMillis)); + ZonedDateTime alignedStart = getAlignedBoundary(type, multiplier, false, shiftedNow); + ZonedDateTime actualStart = alignedStart.plus(Duration.ofMillis(offsetMillis)); + return actualStart.toInstant().toEpochMilli(); } @Override @@ -51,7 +67,11 @@ public abstract class BaseAggInterval implements AggInterval { } protected long getCurrentIntervalEndTs(AggIntervalType type, int multiplier) { - return getAlignedBoundary(type, multiplier, true).toInstant().toEpochMilli(); + ZonedDateTime now = ZonedDateTime.now(); + ZonedDateTime shiftedNow = now.minus(Duration.ofMillis(offsetMillis)); + ZonedDateTime alignedEnd = getAlignedBoundary(type, multiplier, true, shiftedNow); + ZonedDateTime actualEnd = alignedEnd.plus(Duration.ofMillis(offsetMillis)); + return actualEnd.toInstant().toEpochMilli(); } @Override @@ -60,40 +80,32 @@ public abstract class BaseAggInterval implements AggInterval { } protected long getDelayUntilIntervalEnd(AggIntervalType type, int multiplier) { - ZonedDateTime now = ZonedDateTime.now(); - ZonedDateTime currentStart = getAlignedBoundary(type, multiplier, false); - ZonedDateTime nextStart = getAlignedBoundary(type, multiplier, true); - - long periodMillis = Duration.between(currentStart, nextStart).toMillis(); - - // Apply offset: this shifts the grid - long off = offsetMillis % periodMillis; - if (off < 0) off += periodMillis; - - // Compute the offset-aligned start times - ZonedDateTime offsetCurrentStart = currentStart.plus(Duration.ofMillis(off)); - ZonedDateTime offsetNextStart = offsetCurrentStart.plus(Duration.ofMillis(periodMillis)); - - // If we are already past the current offset start, move to the next - ZonedDateTime target = offsetCurrentStart.isAfter(now) ? offsetCurrentStart : offsetNextStart; - // todo fix - return Math.max(Duration.between(now, target).toMillis(), 0); + long currentIntervalEndTs = getCurrentIntervalEndTs(type, multiplier); + long now = System.currentTimeMillis(); + return currentIntervalEndTs - now; } - protected ZonedDateTime getAlignedBoundary(AggIntervalType type, int multiplier, boolean next) { - ZonedDateTime now = ZonedDateTime.now(); - + protected ZonedDateTime getAlignedBoundary(AggIntervalType type, int multiplier, boolean next, ZonedDateTime reference) { return switch (type) { - case HOUR -> alignByHour(now, multiplier, next); - case DAY -> alignByDay(now, multiplier, next); - case WEEK -> alignByWeek(now, multiplier, DayOfWeek.MONDAY, next); - case WEEK_SUN_SAT -> alignByWeek(now, multiplier, DayOfWeek.SUNDAY, next); - case MONTH -> alignByMonth(now, multiplier, next); - case YEAR -> alignByYear(now, multiplier, next); + case MIN -> alignByMin(reference, multiplier, next); + case HOUR -> alignByHour(reference, multiplier, next); + case DAY -> alignByDay(reference, multiplier, next); + case WEEK -> alignByWeek(reference, multiplier, DayOfWeek.MONDAY, next); + case WEEK_SUN_SAT -> alignByWeek(reference, multiplier, DayOfWeek.SUNDAY, next); + case MONTH -> alignByMonth(reference, multiplier, next); + case YEAR -> alignByYear(reference, multiplier, next); default -> throw new IllegalArgumentException("Unsupported type: " + type); }; } + private ZonedDateTime alignByMin(ZonedDateTime now, int multiplier, boolean next) { + ZonedDateTime startOfHour = now.truncatedTo(ChronoUnit.HOURS); + long minsSinceHour = Duration.between(startOfHour, now).toHours(); + long aligned = (minsSinceHour / multiplier) * multiplier; + if (next) aligned += multiplier; + return startOfHour.plusMinutes(aligned); + } + private ZonedDateTime alignByHour(ZonedDateTime now, int multiplier, boolean next) { ZonedDateTime startOfDay = now.truncatedTo(ChronoUnit.DAYS); long hoursSinceMidnight = Duration.between(startOfDay, now).toHours(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java index d875794a18..04a9adc01b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; +import lombok.Data; + +@Data public class CustomInterval extends BaseAggInterval { private int multiplier; // number of base units (e.g. 2 hours, 5 days) @@ -27,7 +30,7 @@ public class CustomInterval extends BaseAggInterval { @Override public long getIntervalDurationMillis() { - return getCurrentIntervalEndTs() - getCurrentIntervalStartTs(); + return getIntervalDurationMillis(internalIntervalType, multiplier); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MinInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MinInterval.java new file mode 100644 index 0000000000..f84cdb9b9e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MinInterval.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; + +import lombok.Data; + +@Data +public class MinInterval extends BaseAggInterval { + + @Override + public AggIntervalType getType() { + return AggIntervalType.HOUR; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/Watermark.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/Watermark.java index 44dd6404f6..52054a12c3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/Watermark.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/Watermark.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; import lombok.Data; diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index bfdce15dbd..fa54685d6e 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -916,6 +916,14 @@ message GeofencingArgumentProto { repeated GeofencingZoneProto zones = 2; } +message ArgumentIntervalProto { + string argName = 1; + int64 startTs = 2; + int64 endTs = 3; + int64 lastArgsRefreshTs = 4; + int64 lastMetricsEvalTs = 5; +} + message CalculatedFieldStateProto { CalculatedFieldEntityCtxIdProto id = 1; string type = 2; @@ -925,6 +933,7 @@ message CalculatedFieldStateProto { AlarmStateProto alarmState = 6; int64 lastArgsUpdateTs = 7; int64 lastMetricsEvalTs = 8; + repeated ArgumentIntervalProto aggregationArguments = 9; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. From 85d83eaa87824a465e07120b5f41db1e989d8326 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 30 Oct 2025 17:00:01 +0200 Subject: [PATCH 04/40] fixes --- .../CalculatedFieldManagerMessageProcessor.java | 6 ++---- .../AbstractCalculatedFieldProcessingService.java | 9 --------- .../DefaultCalculatedFieldProcessingService.java | 6 ++---- .../service/cf/ctx/state/CalculatedFieldCtx.java | 15 --------------- .../EntityAggregationCalculatedFieldState.java | 13 ++++++++++--- .../server/utils/CalculatedFieldUtils.java | 9 +++------ .../aggregation/single/interval/AggInterval.java | 11 ++++++++--- .../single/interval/AggIntervalType.java | 5 ++--- .../single/interval/BaseAggInterval.java | 10 +++++++--- .../single/interval/CustomInterval.java | 11 +++++++++++ .../aggregation/single/interval/DayInterval.java | 2 ++ .../aggregation/single/interval/HourInterval.java | 2 ++ .../aggregation/single/interval/MinInterval.java | 4 +++- .../single/interval/MonthInterval.java | 2 ++ .../aggregation/single/interval/Watermark.java | 4 ++++ .../aggregation/single/interval/WeekInterval.java | 2 ++ .../single/interval/WeekSunSatInterval.java | 2 ++ .../aggregation/single/interval/YearInterval.java | 2 ++ 18 files changed, 64 insertions(+), 51 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 00e1dd3f96..a9d7c0da18 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -305,10 +305,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onRelationChangedEvent(ComponentLifecycleMsg msg, TbCallback callback) { Function> relationAction = switch (msg.getEvent()) { - case RELATION_UPDATED -> - relatedId -> (entityId, ctx, cb) -> initRelatedEntity(entityId, relatedId, ctx, cb); - case RELATION_DELETED -> - relatedId -> (entityId, ctx, cb) -> deleteRelatedEntity(entityId, relatedId, ctx, cb); + case RELATION_UPDATED -> relatedId -> (entityId, ctx, cb) -> initRelatedEntity(entityId, relatedId, ctx, cb); + case RELATION_DELETED -> relatedId -> (entityId, ctx, cb) -> deleteRelatedEntity(entityId, relatedId, ctx, cb); default -> null; }; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 48bce35b3e..c35d0efd5e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -127,15 +127,6 @@ public abstract class AbstractCalculatedFieldProcessingService { return futures; } - private Map> getEntityArgumentsDuringInterval(CalculatedFieldCtx ctx, EntityId entityId, long ts) { - Map> futures = new HashMap<>(); - for (var entry : ctx.getArguments().entrySet()) { - var argValueFuture = fetchArgumentValue(ctx.getTenantId(), entityId, entry.getValue(), ts); - futures.put(entry.getKey(), argValueFuture); - } - return futures; - } - protected EntityId resolveEntityId(TenantId tenantId, EntityId entityId, Argument argument) { if (argument.getRefEntityId() != null) { return argument.getRefEntityId(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index c6a66d100e..10ace6b8cf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -92,10 +92,8 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF @Override public Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId) { return switch (ctx.getCfType()) { - case GEOFENCING -> - resolveArgumentFutures(fetchGeofencingCalculatedFieldArguments(ctx, entityId, true, System.currentTimeMillis())); - case PROPAGATION -> - resolveArgumentFutures(Map.of(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId))); + case GEOFENCING -> resolveArgumentFutures(fetchGeofencingCalculatedFieldArguments(ctx, entityId, true, System.currentTimeMillis())); + case PROPAGATION -> resolveArgumentFutures(Map.of(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId))); default -> Collections.emptyMap(); }; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index c0208825a8..92895617c1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -206,21 +206,6 @@ public class CalculatedFieldCtx implements Closeable { this.maxStateSize = systemContext.getApiLimitService().getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024; this.maxSingleValueArgumentSize = systemContext.getApiLimitService().getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxSingleValueArgumentSizeInKBytes) * 1024; } -// -// public boolean isRequiresScheduledReevaluation() { -// if (CalculatedFieldType.ENTITY_AGGREGATION.equals(calculatedField.getType())) { -// var configuration = (EntityAggregationCalculatedFieldConfiguration) calculatedField.getConfiguration(); -// AggInterval interval = configuration.getInterval(); -// long delayUntilIntervalEnd = interval.getDelayUntilIntervalEnd(); -// if (lastReevaluationTs < System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(systemContext.getCfCheckInterval())) { -// -// } -// if (TimeUnit.SECONDS.toMillis(systemContext.getCfCheckInterval()) >= delayUntilIntervalEnd) { -// return true; -// } -// } -// return requiresScheduledReevaluation; -// } public void init() { switch (cfType) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java index 33a8003e03..93d83476cc 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java @@ -35,8 +35,10 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.BaseCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; +import java.util.List; import java.util.Map; public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldState { @@ -117,11 +119,13 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt long now = System.currentTimeMillis(); Map> results = new HashMap<>(); + List expiredIntervals = new ArrayList<>(); for (Map.Entry> entry : intervals.entrySet()) { AggIntervalEntry intervalEntry = entry.getKey(); Map args = entry.getValue(); - processInterval(now, intervalEntry, args, results); + processInterval(now, intervalEntry, args, expiredIntervals, results); } + expiredIntervals.forEach(intervals::remove); ArrayNode result = toResult(results); if (result.isEmpty()) { @@ -157,14 +161,17 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt ctx.scheduleReevaluation(interval.getDelayUntilIntervalEnd(), actorCtx); } - private void processInterval(long now, AggIntervalEntry intervalEntry, Map args, + private void processInterval(long now, + AggIntervalEntry intervalEntry, + Map args, + List expiredIntervals, Map> results) throws Exception { long startTs = intervalEntry.getStartTs(); long endTs = intervalEntry.getEndTs(); if (now - endTs > watermarkDuration) { handleExpiredInterval(intervalEntry, args, results); - intervals.remove(intervalEntry); + expiredIntervals.add(intervalEntry); } else if (now - startTs >= intervalDuration) { handleActiveInterval(intervalEntry, args, results); } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 780df5b795..710ce48ef6 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -102,12 +102,9 @@ public class CalculatedFieldUtils { state.getArguments().forEach((argName, argEntry) -> { switch (argEntry.getType()) { - case SINGLE_VALUE -> - builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); - case TS_ROLLING -> - builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry)); - case GEOFENCING -> - builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); + case SINGLE_VALUE -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); + case TS_ROLLING -> builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry)); + case GEOFENCING -> builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); case RELATED_ENTITIES -> { RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry; relatedEntitiesArgumentEntry.getEntityInputs() diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java index 4c21c80086..d923eb5d7a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -25,27 +26,31 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; property = "type" ) @JsonSubTypes({ + @JsonSubTypes.Type(value = MinInterval.class, name = "MIN"), @JsonSubTypes.Type(value = HourInterval.class, name = "HOUR"), @JsonSubTypes.Type(value = DayInterval.class, name = "DAY"), @JsonSubTypes.Type(value = WeekInterval.class, name = "WEEK"), @JsonSubTypes.Type(value = WeekSunSatInterval.class, name = "WEEK_SUN_SAT"), @JsonSubTypes.Type(value = MonthInterval.class, name = "MONTH"), @JsonSubTypes.Type(value = YearInterval.class, name = "YEAR"), - @JsonSubTypes.Type(value = CustomInterval.class, name = "CUSTOM"), - - @JsonSubTypes.Type(value = MinInterval.class, name = "MIN")// todo: delete. used only for tests + @JsonSubTypes.Type(value = CustomInterval.class, name = "CUSTOM") }) @JsonIgnoreProperties(ignoreUnknown = true) public interface AggInterval { + @JsonIgnore AggIntervalType getType(); + @JsonIgnore long getIntervalDurationMillis(); + @JsonIgnore long getCurrentIntervalStartTs(); + @JsonIgnore long getCurrentIntervalEndTs(); + @JsonIgnore long getDelayUntilIntervalEnd(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java index 82cf46c6ff..96bac69128 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java @@ -17,14 +17,13 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.i public enum AggIntervalType { + MIN, HOUR, DAY, WEEK, WEEK_SUN_SAT, MONTH, YEAR, - CUSTOM, - - MIN// todo: delete + CUSTOM } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java index cc207133fa..d8df2d9826 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java @@ -22,6 +22,7 @@ import java.time.Duration; import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime; +import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.temporal.ChronoUnit; import java.time.temporal.TemporalAdjusters; @@ -29,6 +30,7 @@ import java.time.temporal.TemporalAdjusters; @Data public abstract class BaseAggInterval implements AggInterval { + protected String tz; protected long offsetMillis; // delay millis since start of interval @Override @@ -54,7 +56,8 @@ public abstract class BaseAggInterval implements AggInterval { } protected long getCurrentIntervalStartTs(AggIntervalType type, int multiplier) { - ZonedDateTime now = ZonedDateTime.now(); + ZoneId zoneId = ZoneId.of(tz); + ZonedDateTime now = ZonedDateTime.now(zoneId); ZonedDateTime shiftedNow = now.minus(Duration.ofMillis(offsetMillis)); ZonedDateTime alignedStart = getAlignedBoundary(type, multiplier, false, shiftedNow); ZonedDateTime actualStart = alignedStart.plus(Duration.ofMillis(offsetMillis)); @@ -67,7 +70,8 @@ public abstract class BaseAggInterval implements AggInterval { } protected long getCurrentIntervalEndTs(AggIntervalType type, int multiplier) { - ZonedDateTime now = ZonedDateTime.now(); + ZoneId zoneId = ZoneId.of(tz); + ZonedDateTime now = ZonedDateTime.now(zoneId); ZonedDateTime shiftedNow = now.minus(Duration.ofMillis(offsetMillis)); ZonedDateTime alignedEnd = getAlignedBoundary(type, multiplier, true, shiftedNow); ZonedDateTime actualEnd = alignedEnd.plus(Duration.ofMillis(offsetMillis)); @@ -100,7 +104,7 @@ public abstract class BaseAggInterval implements AggInterval { private ZonedDateTime alignByMin(ZonedDateTime now, int multiplier, boolean next) { ZonedDateTime startOfHour = now.truncatedTo(ChronoUnit.HOURS); - long minsSinceHour = Duration.between(startOfHour, now).toHours(); + long minsSinceHour = Duration.between(startOfHour, now).toMinutes(); long aligned = (minsSinceHour / multiplier) * multiplier; if (next) aligned += multiplier; return startOfHour.plusMinutes(aligned); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java index 04a9adc01b..cc56b2aef3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java @@ -16,13 +16,24 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +@EqualsAndHashCode(callSuper = true) @Data +@NoArgsConstructor public class CustomInterval extends BaseAggInterval { private int multiplier; // number of base units (e.g. 2 hours, 5 days) private AggIntervalType internalIntervalType; + public CustomInterval(int multiplier, AggIntervalType internalIntervalType, long offsetMillis, String tz) { + this.tz = tz; + this.offsetMillis = offsetMillis; + this.multiplier = multiplier; + this.internalIntervalType = internalIntervalType; + } + @Override public AggIntervalType getType() { return AggIntervalType.CUSTOM; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java index 01cbdf2e97..e5f48d3116 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java @@ -16,8 +16,10 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; import lombok.Data; +import lombok.NoArgsConstructor; @Data +@NoArgsConstructor public class DayInterval extends BaseAggInterval { @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java index ce84b57ae1..dfd7b7efda 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java @@ -16,8 +16,10 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; import lombok.Data; +import lombok.NoArgsConstructor; @Data +@NoArgsConstructor public class HourInterval extends BaseAggInterval { @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MinInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MinInterval.java index f84cdb9b9e..066bc230b8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MinInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MinInterval.java @@ -16,13 +16,15 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; import lombok.Data; +import lombok.NoArgsConstructor; @Data +@NoArgsConstructor public class MinInterval extends BaseAggInterval { @Override public AggIntervalType getType() { - return AggIntervalType.HOUR; + return AggIntervalType.MIN; } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java index fe8d60f41c..7225eaca9f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java @@ -16,8 +16,10 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; import lombok.Data; +import lombok.NoArgsConstructor; @Data +@NoArgsConstructor public class MonthInterval extends BaseAggInterval { @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/Watermark.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/Watermark.java index 52054a12c3..20d03194ed 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/Watermark.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/Watermark.java @@ -15,9 +15,13 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; +import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; @Data +@AllArgsConstructor +@NoArgsConstructor public class Watermark { private long duration; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java index 5a93076772..2ee5d5f81c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java @@ -16,8 +16,10 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; import lombok.Data; +import lombok.NoArgsConstructor; @Data +@NoArgsConstructor public class WeekInterval extends BaseAggInterval { @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java index c70dd79a9f..f2d403c173 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java @@ -16,8 +16,10 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; import lombok.Data; +import lombok.NoArgsConstructor; @Data +@NoArgsConstructor public class WeekSunSatInterval extends BaseAggInterval { @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java index 3c600064d1..23aedf4932 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java @@ -16,8 +16,10 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; import lombok.Data; +import lombok.NoArgsConstructor; @Data +@NoArgsConstructor public class YearInterval extends BaseAggInterval { @Override From 456e92c45dc9a7f00fca1364b6cedb1165ff7b1d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 31 Oct 2025 12:39:10 +0200 Subject: [PATCH 05/40] refactoring --- ...tractCalculatedFieldProcessingService.java | 79 ++++++++----------- .../utils/CalculatedFieldArgumentUtils.java | 24 +++++- 2 files changed, 56 insertions(+), 47 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index c35d0efd5e..4e2e81e9be 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -54,8 +54,6 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntryStatus; -import org.thingsboard.server.service.cf.ctx.state.aggregation.single.EntityAggregationArgumentEntry; import java.util.Collections; import java.util.HashMap; @@ -64,7 +62,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.cf.CalculatedFieldType.PROPAGATION; @@ -73,7 +71,9 @@ import static org.thingsboard.server.common.data.cf.configuration.geofencing.Ent import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LONGITUDE_ARGUMENT_KEY; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultAttributeEntry; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultKvEntry; +import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformAggregationArgument; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformSingleValueArgument; +import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformTsRollingArgument; @Data @Slf4j @@ -141,19 +141,21 @@ public abstract class AbstractCalculatedFieldProcessingService { return argFutures.entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, // Keep the key as is - entry -> { - try { - return entry.getValue().get(); - } catch (ExecutionException e) { - Throwable cause = e.getCause(); - throw new RuntimeException("Failed to fetch " + entry.getKey() + ": " + cause.getMessage(), cause); - } catch (InterruptedException e) { - throw new RuntimeException("Failed to fetch" + entry.getKey(), e); - } - } + entry -> resolveArgumentValue(entry.getKey(), entry.getValue()) )); } + protected ArgumentEntry resolveArgumentValue(String argName, ListenableFuture future) { + try { + return future.get(); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + throw new RuntimeException("Failed to fetch " + argName + ": " + cause.getMessage(), cause); + } catch (InterruptedException e) { + throw new RuntimeException("Failed to fetch" + argName, e); + } + } + protected ListenableFuture fetchPropagationCalculatedFieldArgument(CalculatedFieldCtx ctx, EntityId entityId) { ListenableFuture> propagationEntityIds = fromDynamicSource(ctx.getTenantId(), entityId, ctx.getPropagationArgument()); return Futures.transform(propagationEntityIds, ArgumentEntry::createPropagationArgument, MoreExecutors.directExecutor()); @@ -199,7 +201,7 @@ public abstract class AbstractCalculatedFieldProcessingService { return aggConfig.getArguments().entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, - entry -> fetchTimeSeries(ctx.getTenantId(), entityId, entry.getValue(), aggConfig.getInterval()) + entry -> fetchTimeSeries(ctx.getTenantId(), entityId, entry.getValue(), aggConfig.getInterval(), ts) )); } @@ -319,45 +321,21 @@ public abstract class AbstractCalculatedFieldProcessingService { return ArgumentEntry.createSingleValueArgument(timeSeries.get(0)); }, calculatedFieldCallbackExecutor); - // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. - // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. - // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, - // but this will significantly complicate the code. - return argumentEntryFut.get(1, TimeUnit.MINUTES); + return resolveArgumentValue(argName, argumentEntryFut); } - private ListenableFuture fetchTimeSeries(TenantId tenantId, EntityId entityId, Argument argument, AggInterval interval) { + private ListenableFuture fetchTimeSeries(TenantId tenantId, EntityId entityId, Argument argument, AggInterval interval, long queryEndTs) { long startInterval = interval.getCurrentIntervalStartTs(); - - String key = argument.getRefEntityKey().getKey(); - ReadTsKvQuery query = new BaseReadTsKvQuery(key, startInterval, System.currentTimeMillis(), 0, 1, Aggregation.NONE); - - log.trace("[{}][{}] Fetching timeseries for query {}", tenantId, entityId, query); - ListenableFuture> fetchedTelemetryFut = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(fetchedTelemetryFut, telemetry -> { - log.debug("[{}][{}] Fetched {} timeseries for query {}", tenantId, entityId, telemetry == null ? 0 : telemetry.size(), query); - Map aggIntervals = new HashMap<>(); - AggIntervalEntry aggIntervalEntry = new AggIntervalEntry(interval.getCurrentIntervalStartTs(), interval.getCurrentIntervalEndTs()); - if (telemetry == null || telemetry.isEmpty()) { - aggIntervals.put(aggIntervalEntry, new AggIntervalEntryStatus()); - } else { - aggIntervals.put(aggIntervalEntry, new AggIntervalEntryStatus(System.currentTimeMillis())); - } - return new EntityAggregationArgumentEntry(aggIntervals); - }, calculatedFieldCallbackExecutor); + long intervalEndTs = interval.getCurrentIntervalEndTs(); + ReadTsKvQuery query = buildTimeSeriesQuery(tenantId, argument, startInterval, queryEndTs); + return fetchTimeSeriesInternal(tenantId, entityId, query, timeSeries -> transformAggregationArgument(timeSeries, startInterval, intervalEndTs)); } private ListenableFuture fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument, long queryEndTs) { long argTimeWindow = argument.getTimeWindow() == 0 ? queryEndTs : argument.getTimeWindow(); long startInterval = queryEndTs - argTimeWindow; - ReadTsKvQuery query = buildTsRollingQuery(tenantId, argument, startInterval, queryEndTs); - - log.trace("[{}][{}] Fetching timeseries for query {}", tenantId, entityId, query); - ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(tsRollingFuture, tsRolling -> { - log.debug("[{}][{}] Fetched {} timeseries for query {}", tenantId, entityId, tsRolling == null ? 0 : tsRolling.size(), query); - return ArgumentEntry.createTsRollingArgument(tsRolling, query.getLimit(), argTimeWindow); - }, calculatedFieldCallbackExecutor); + ReadTsKvQuery query = buildTimeSeriesQuery(tenantId, argument, startInterval, queryEndTs); + return fetchTimeSeriesInternal(tenantId, entityId, query, tsRolling -> transformTsRollingArgument(tsRolling, query.getLimit(), argTimeWindow)); } private ListenableFuture fetchAttribute(TenantId tenantId, EntityId entityId, Argument argument, long defaultLastUpdateTs) { @@ -383,7 +361,16 @@ public abstract class AbstractCalculatedFieldProcessingService { }, calculatedFieldCallbackExecutor)); } - private ReadTsKvQuery buildTsRollingQuery(TenantId tenantId, Argument argument, long startTs, long endTs) { + private ListenableFuture fetchTimeSeriesInternal(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, Function, ArgumentEntry> transformArgument) { + log.trace("[{}][{}] Fetching timeseries for query {}", tenantId, entityId, query); + ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); + return Futures.transform(tsRollingFuture, tsRolling -> { + log.debug("[{}][{}] Fetched {} timeseries for query {}", tenantId, entityId, tsRolling == null ? 0 : tsRolling.size(), query); + return transformArgument.apply(tsRolling); + }, calculatedFieldCallbackExecutor); + } + + private ReadTsKvQuery buildTimeSeriesQuery(TenantId tenantId, Argument argument, long startTs, long endTs) { long maxDataPoints = apiLimitService.getLimit( tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); int argumentLimit = argument.getLimit(); diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index 72b5b73471..0477669663 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -28,18 +28,25 @@ import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.single.EntityAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntryStatus; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.EntityAggregationArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.EntityAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalculatedFieldState; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Optional; public class CalculatedFieldArgumentUtils { @@ -56,6 +63,21 @@ public class CalculatedFieldArgumentUtils { } } + public static ArgumentEntry transformTsRollingArgument(List tsRolling, int limit, long argTimeWindow) { + return ArgumentEntry.createTsRollingArgument(tsRolling, limit, argTimeWindow); + } + + public static ArgumentEntry transformAggregationArgument(List telemetry, long startIntervalTs, long endIntervalTs) { + Map aggIntervals = new HashMap<>(); + AggIntervalEntry aggIntervalEntry = new AggIntervalEntry(startIntervalTs, endIntervalTs); + if (telemetry == null || telemetry.isEmpty()) { + aggIntervals.put(aggIntervalEntry, new AggIntervalEntryStatus()); + } else { + aggIntervals.put(aggIntervalEntry, new AggIntervalEntryStatus(System.currentTimeMillis())); + } + return new EntityAggregationArgumentEntry(aggIntervals); + } + public static KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getRefEntityKey().getKey(); String defaultValue = argument.getDefaultValue(); From f1ce07ef3f9522a5e9e36c33425c1c4d2e3896a5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 31 Oct 2025 13:54:28 +0200 Subject: [PATCH 06/40] refactoring --- ...tractCalculatedFieldProcessingService.java | 34 +++++------------- .../cf/CalculatedFieldProcessingService.java | 3 +- ...faultCalculatedFieldProcessingService.java | 5 +-- ...EntityAggregationCalculatedFieldState.java | 36 +++++++++---------- .../utils/CalculatedFieldArgumentUtils.java | 11 ++++-- 5 files changed, 38 insertions(+), 51 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 4e2e81e9be..c58444397c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -28,7 +28,6 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; -import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.aggregation.single.EntityAggregationCalculatedFieldConfiguration; @@ -54,6 +53,7 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntry; +import org.thingsboard.server.utils.CalculatedFieldArgumentUtils; import java.util.Collections; import java.util.HashMap; @@ -145,14 +145,14 @@ public abstract class AbstractCalculatedFieldProcessingService { )); } - protected ArgumentEntry resolveArgumentValue(String argName, ListenableFuture future) { + protected ArgumentEntry resolveArgumentValue(String key, ListenableFuture future) { try { return future.get(); } catch (ExecutionException e) { Throwable cause = e.getCause(); - throw new RuntimeException("Failed to fetch " + argName + ": " + cause.getMessage(), cause); + throw new RuntimeException("Failed to fetch " + key + ": " + cause.getMessage(), cause); } catch (InterruptedException e) { - throw new RuntimeException("Failed to fetch" + argName, e); + throw new RuntimeException("Failed to fetch" + key, e); } } @@ -298,30 +298,12 @@ public abstract class AbstractCalculatedFieldProcessingService { }; } - protected ArgumentEntry fetchMetricDuringInterval(EntityId entityId, AggIntervalEntry interval, String metricName, CalculatedFieldCtx ctx) throws Exception { - var config = (EntityAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - - AggMetric metric = config.getMetrics().get(metricName); + protected ArgumentEntry fetchMetricDuringInterval(TenantId tenantId, EntityId entityId, String argKey, AggMetric metric, AggIntervalEntry interval) { AggFunction function = metric.getFunction(); - - AggKeyInput input = (AggKeyInput) metric.getInput(); - String argName = input.getKey(); - Argument argument = ctx.getArguments().get(argName); - String key = argument.getRefEntityKey().getKey(); - long intervalMs = interval.getEndTs() - interval.getStartTs(); - BaseReadTsKvQuery query = new BaseReadTsKvQuery(key, interval.getStartTs(), interval.getEndTs(), intervalMs, 1, Aggregation.valueOf(function.name())); - log.trace("[{}][{}] Fetching timeseries for query {}", ctx.getTenantId(), entityId, query); - ListenableFuture> tsFuture = timeseriesService.findAll(ctx.getTenantId(), entityId, List.of(query)); - ListenableFuture argumentEntryFut = Futures.transform(tsFuture, timeSeries -> { - log.debug("[{}][{}] Fetched {} timeseries for query {}", ctx.getTenantId(), entityId, timeSeries == null ? 0 : timeSeries.size(), query); - if (timeSeries == null || timeSeries.isEmpty()) { - return new SingleValueArgumentEntry(); - } - return ArgumentEntry.createSingleValueArgument(timeSeries.get(0)); - }, calculatedFieldCallbackExecutor); - - return resolveArgumentValue(argName, argumentEntryFut); + BaseReadTsKvQuery query = new BaseReadTsKvQuery(argKey, interval.getStartTs(), interval.getEndTs(), intervalMs, 1, Aggregation.valueOf(function.name())); + ListenableFuture argumentEntryFut = fetchTimeSeriesInternal(tenantId, entityId, query, CalculatedFieldArgumentUtils::transformAggMetricArgument); + return resolveArgumentValue(argKey, argumentEntryFut); } private ListenableFuture fetchTimeSeries(TenantId tenantId, EntityId entityId, Argument argument, AggInterval interval, long queryEndTs) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index e65df09f98..67bd2fec0c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.cf; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -38,7 +39,7 @@ public interface CalculatedFieldProcessingService { Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments); - ArgumentEntry fetchMetricDuringInterval(EntityId entityId, AggIntervalEntry interval, String argName, CalculatedFieldCtx ctx) throws Exception; + ArgumentEntry fetchMetricDuringInterval(TenantId tenantId, EntityId entityId, String argKey, AggMetric metric, AggIntervalEntry interval); void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult result, List cfIds, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index 10ace6b8cf..c0114780b2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -113,8 +114,8 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF } @Override - public ArgumentEntry fetchMetricDuringInterval(EntityId entityId, AggIntervalEntry interval, String metricName, CalculatedFieldCtx ctx) throws Exception { - return super.fetchMetricDuringInterval(entityId, interval, metricName, ctx); + public ArgumentEntry fetchMetricDuringInterval(TenantId tenantId, EntityId entityId, String argKey, AggMetric metric, AggIntervalEntry interval) { + return super.fetchMetricDuringInterval(tenantId, entityId, argKey, metric, interval); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java index 93d83476cc..a5daf36b58 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java @@ -120,11 +120,9 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt Map> results = new HashMap<>(); List expiredIntervals = new ArrayList<>(); - for (Map.Entry> entry : intervals.entrySet()) { - AggIntervalEntry intervalEntry = entry.getKey(); - Map args = entry.getValue(); - processInterval(now, intervalEntry, args, expiredIntervals, results); - } + intervals.forEach((intervalEntry, argIntervalStatuses) -> { + processInterval(now, intervalEntry, argIntervalStatuses, expiredIntervals, results); + }); expiredIntervals.forEach(intervals::remove); ArrayNode result = toResult(results); @@ -165,7 +163,7 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt AggIntervalEntry intervalEntry, Map args, List expiredIntervals, - Map> results) throws Exception { + Map> results) { long startTs = intervalEntry.getStartTs(); long endTs = intervalEntry.getEndTs(); @@ -179,37 +177,35 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt private void handleExpiredInterval(AggIntervalEntry intervalEntry, Map args, - Map> results) throws Exception { - for (Map.Entry argStatus : args.entrySet()) { - String argName = argStatus.getKey(); - AggIntervalEntryStatus argEntryIntervalStatus = argStatus.getValue(); + Map> results) { + args.forEach((argName, argEntryIntervalStatus) -> { if (argEntryIntervalStatus.getLastArgsRefreshTs() > argEntryIntervalStatus.getLastMetricsEvalTs()) { processMetric(intervalEntry, argName, results); } - } + }); } private void handleActiveInterval(AggIntervalEntry intervalEntry, Map args, - Map> results) throws Exception { - for (Map.Entry argStatus : args.entrySet()) { - String argName = argStatus.getKey(); - AggIntervalEntryStatus argEntryIntervalStatus = argStatus.getValue(); + Map> results) { + args.forEach((argName, argEntryIntervalStatus) -> { if (argEntryIntervalStatus.shouldRecalculate(checkInterval)) { processMetric(intervalEntry, argName, results); ctx.scheduleReevaluation(checkInterval, actorCtx); } - } + }); } private void processMetric(AggIntervalEntry intervalEntry, String argName, - Map> results) throws Exception { + Map> results) { String metricName = findMetricName(argName); if (metricName != null) { - ArgumentEntry metric = cfProcessingService.fetchMetricDuringInterval(entityId, intervalEntry, metricName, ctx); - if (!metric.isEmpty()) { - results.computeIfAbsent(intervalEntry, i -> new HashMap<>()).put(metricName, metric); + AggMetric metric = metrics.get(metricName); + String argKey = ctx.getArguments().get(argName).getRefEntityKey().getKey(); + ArgumentEntry metricEntry = cfProcessingService.fetchMetricDuringInterval(ctx.getTenantId(), entityId, argKey, metric, intervalEntry); + if (!metricEntry.isEmpty()) { + results.computeIfAbsent(intervalEntry, i -> new HashMap<>()).put(metricName, metricEntry); } } } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index 0477669663..c6c64782a9 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -67,10 +67,17 @@ public class CalculatedFieldArgumentUtils { return ArgumentEntry.createTsRollingArgument(tsRolling, limit, argTimeWindow); } - public static ArgumentEntry transformAggregationArgument(List telemetry, long startIntervalTs, long endIntervalTs) { + public static ArgumentEntry transformAggMetricArgument(List timeSeries) { + if (timeSeries == null || timeSeries.isEmpty()) { + return new SingleValueArgumentEntry(); + } + return ArgumentEntry.createSingleValueArgument(timeSeries.get(0)); + } + + public static ArgumentEntry transformAggregationArgument(List timeSeries, long startIntervalTs, long endIntervalTs) { Map aggIntervals = new HashMap<>(); AggIntervalEntry aggIntervalEntry = new AggIntervalEntry(startIntervalTs, endIntervalTs); - if (telemetry == null || telemetry.isEmpty()) { + if (timeSeries == null || timeSeries.isEmpty()) { aggIntervals.put(aggIntervalEntry, new AggIntervalEntryStatus()); } else { aggIntervals.put(aggIntervalEntry, new AggIntervalEntryStatus(System.currentTimeMillis())); From d89ce673ab6eb8d014fd2e56665202a9ad459718 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 31 Oct 2025 14:27:09 +0200 Subject: [PATCH 07/40] fixed interval removal after expiration --- .../single/EntityAggregationCalculatedFieldState.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java index a5daf36b58..49d17a6a9f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java @@ -123,7 +123,7 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt intervals.forEach((intervalEntry, argIntervalStatuses) -> { processInterval(now, intervalEntry, argIntervalStatuses, expiredIntervals, results); }); - expiredIntervals.forEach(intervals::remove); + removeExpiredIntervals(expiredIntervals); ArrayNode result = toResult(results); if (result.isEmpty()) { @@ -146,6 +146,15 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt }); } + private void removeExpiredIntervals(List expiredIntervals) { + expiredIntervals.forEach(expiredInterval -> { + arguments.values().stream() + .map(EntityAggregationArgumentEntry.class::cast) + .forEach(arg -> arg.getAggIntervals().remove(expiredInterval)); + intervals.remove(expiredInterval); + }); + } + private void createIntervalIfNotExist() { AggIntervalEntry currentInterval = new AggIntervalEntry(interval.getCurrentIntervalStartTs(), interval.getCurrentIntervalEndTs()); if (intervals.containsKey(currentInterval)) { From c3489e4047fed693b2ca98e1f41f8093d35783d9 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 31 Oct 2025 16:32:24 +0200 Subject: [PATCH 08/40] changed limit --- .../cf/AbstractCalculatedFieldProcessingService.java | 2 +- .../single/EntityAggregationCalculatedFieldState.java | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index c58444397c..1b13bccda1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -355,7 +355,7 @@ public abstract class AbstractCalculatedFieldProcessingService { private ReadTsKvQuery buildTimeSeriesQuery(TenantId tenantId, Argument argument, long startTs, long endTs) { long maxDataPoints = apiLimitService.getLimit( tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); - int argumentLimit = argument.getLimit(); + int argumentLimit = argument.getLimit() == null ? 500000 : argument.getLimit(); int limit = argumentLimit == 0 || argumentLimit > maxDataPoints ? (int) maxDataPoints : argumentLimit; return new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, endTs, 0, limit, Aggregation.NONE); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java index 49d17a6a9f..6885aa8b42 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java @@ -86,8 +86,9 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt arguments.forEach((argName, argumentEntry) -> { var entityAggEntry = (EntityAggregationArgumentEntry) argumentEntry; - entityAggEntry.getAggIntervals().put(missingAggIntervalEntry, new AggIntervalEntryStatus()); - intervals.computeIfAbsent(missingAggIntervalEntry, i -> new HashMap<>()).put(argName, new AggIntervalEntryStatus()); + AggIntervalEntryStatus intervalEntryStatus = new AggIntervalEntryStatus(System.currentTimeMillis()); + entityAggEntry.getAggIntervals().put(missingAggIntervalEntry, intervalEntryStatus); + intervals.computeIfAbsent(missingAggIntervalEntry, i -> new HashMap<>()).put(argName, intervalEntryStatus); }); nextStartTs = nextEndTs; @@ -114,8 +115,8 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception { - createIntervalIfNotExist(); prepareIntervals(); + createIntervalIfNotExist(); long now = System.currentTimeMillis(); Map> results = new HashMap<>(); From a394f4015acee760934c1ecb3209169c428bbe03 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 31 Oct 2025 20:18:29 +0200 Subject: [PATCH 09/40] UI: New calculated field entity aggregation --- .../calculated-field.module.ts | 4 + ...ulated-field-argument-panel.component.html | 70 +++--- ...lculated-field-argument-panel.component.ts | 2 + ...calculated-field-arguments-table.module.ts | 9 +- ...y-aggregation-arguments-table.component.ts | 71 ++++++ .../calculated-field-dialog.component.html | 7 + ...ntity-aggregation-component.component.html | 131 +++++++++++ .../entity-aggregation-component.component.ts | 205 ++++++++++++++++++ .../entity-aggregation-component.module.ts | 49 +++++ ...culated-field-metrics-panel.component.html | 84 +++---- ...alculated-field-metrics-panel.component.ts | 1 + ...culated-field-metrics-table.component.html | 20 +- ...alculated-field-metrics-table.component.ts | 18 +- .../calculated-field-metrics-table.module.ts | 27 +++ .../calculated-field-output.component.html | 4 +- .../calculated-field-output.component.ts | 13 ++ ...ities-aggregation-component.component.html | 1 + ...d-entities-aggregation-component.module.ts | 10 +- .../components/time-unit-input.component.html | 6 +- .../components/time-unit-input.component.ts | 7 + .../shared/models/calculated-field.models.ts | 50 ++++- .../assets/locale/locale.constant-en_US.json | 34 ++- 22 files changed, 725 insertions(+), 98 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/entity-aggregation-arguments-table.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.module.ts rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{related-entities-aggregation-configuration => metrics}/calculated-field-metrics-panel.component.html (73%) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{related-entities-aggregation-configuration => metrics}/calculated-field-metrics-panel.component.ts (99%) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{related-entities-aggregation-configuration => metrics}/calculated-field-metrics-table.component.html (91%) rename ui-ngx/src/app/modules/home/components/calculated-fields/components/{related-entities-aggregation-configuration => metrics}/calculated-field-metrics-table.component.ts (94%) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-table.module.ts diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts index 5e3a854aa2..e9bf146bfd 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts @@ -41,6 +41,9 @@ import { import { RelatedEntitiesAggregationComponentModule } from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module'; +import { + EntityAggregationComponentModule +} from '@home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.module'; @NgModule({ declarations: [ @@ -56,6 +59,7 @@ import { SimpleConfigurationModule, PropagationConfigurationModule, RelatedEntitiesAggregationComponentModule, + EntityAggregationComponentModule, ], exports: [ CalculatedFieldDialogComponent, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html index c7eeaa253b..459c9a4dbb 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html @@ -74,25 +74,27 @@ } -
-
{{ 'calculated-fields.argument-type' | translate }}
- - - @for (type of argumentTypes; track type) { - {{ ArgumentTypeTranslations.get(type) | translate }} + @if (!hiddenEntityKeyTypes) { +
+
{{ 'calculated-fields.argument-type' | translate }}
+ + + @for (type of argumentTypes; track type) { + {{ ArgumentTypeTranslations.get(type) | translate }} + } + + @if (refEntityKeyFormGroup.get('type').hasError('required') && refEntityKeyFormGroup.get('type').touched) { + + warning + } - - @if (refEntityKeyFormGroup.get('type').hasError('required') && refEntityKeyFormGroup.get('type').touched) { - - warning - - } - -
+ +
+ } @if (entityFilter.singleEntity?.id || entityType === ArgumentEntityType.Current || entityType === ArgumentEntityType.Tenant) { @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) {
@@ -149,21 +151,23 @@ }"> } @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Rolling) { -
-
{{ 'calculated-fields.default-value' | translate }}
- - - @if (argumentFormGroup.get('defaultValue').touched && argumentFormGroup.get('defaultValue').hasError('required')) { - - warning - - } - -
+ @if (!hiddenDefaultValue) { +
+
{{ 'calculated-fields.default-value' | translate }}
+ + + @if (argumentFormGroup.get('defaultValue').touched && argumentFormGroup.get('defaultValue').hasError('required')) { + + warning + + } + +
+ } } @else {
{{ 'calculated-fields.time-window' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts index dcc54c94b3..e58145cf05 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts @@ -69,6 +69,8 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI @Input() usedArgumentNames: string[]; @Input() isOutputKey = false; @Input() hiddenEntityTypes = false; + @Input() hiddenEntityKeyTypes = false; + @Input() hiddenDefaultValue = false; @Input() defaultValueRequired = false; @Input() hint: string; @Input() predefinedEntityFilter: EntityFilter; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts index cae0b92387..494d6f0159 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts @@ -29,6 +29,9 @@ import { import { RelatedAggregationArgumentsTableComponent } from '@home/components/calculated-fields/components/calculated-field-arguments/related-aggregation-arguments-table.component'; +import { + EntityAggregationArgumentsTableComponent +} from '@home/components/calculated-fields/components/calculated-field-arguments/entity-aggregation-arguments-table.component'; @NgModule({ imports: [ @@ -39,12 +42,14 @@ import { CalculatedFieldArgumentPanelComponent, CalculatedFieldArgumentsTableComponent, PropagateArgumentsTableComponent, - RelatedAggregationArgumentsTableComponent + RelatedAggregationArgumentsTableComponent, + EntityAggregationArgumentsTableComponent, ], exports: [ CalculatedFieldArgumentsTableComponent, PropagateArgumentsTableComponent, - RelatedAggregationArgumentsTableComponent + RelatedAggregationArgumentsTableComponent, + EntityAggregationArgumentsTableComponent, ] }) export class CalculatedFieldArgumentsTableModule {} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/entity-aggregation-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/entity-aggregation-arguments-table.component.ts new file mode 100644 index 0000000000..6d9b746d59 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/entity-aggregation-arguments-table.component.ts @@ -0,0 +1,71 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { ChangeDetectorRef, Component, DestroyRef, forwardRef, Renderer2, ViewContainerRef, } from '@angular/core'; +import { FormBuilder, NG_VALIDATORS, NG_VALUE_ACCESSOR, } from '@angular/forms'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityService } from '@core/http/entity.service'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { + CalculatedFieldArgumentsTableComponent +} from '@home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.component'; +import { ArgumentEntityType } from '@shared/models/calculated-field.models'; + +@Component({ + selector: 'tb-entity-aggregation-arguments-table', + templateUrl: './calculated-field-arguments-table.component.html', + styleUrls: [`calculated-field-arguments-table.component.scss`], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => EntityAggregationArgumentsTableComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => EntityAggregationArgumentsTableComponent), + multi: true + } + ], +}) +export class EntityAggregationArgumentsTableComponent extends CalculatedFieldArgumentsTableComponent { + + constructor( + protected fb: FormBuilder, + protected popoverService: TbPopoverService, + protected viewContainerRef: ViewContainerRef, + protected cd: ChangeDetectorRef, + protected renderer: Renderer2, + protected entityService: EntityService, + protected destroyRef: DestroyRef, + protected store: Store + ) { + super(fb, popoverService, viewContainerRef, cd, renderer, entityService, destroyRef, store); + + this.argumentNameColumn = 'calculated-fields.argument-name'; + this.displayColumns = ['name', 'type', 'key', 'actions']; + this.panelAdditionalCtx = { + hiddenEntityTypes: true, + argumentEntityTypes: [ArgumentEntityType.Current], + hint: 'calculated-fields.entity-aggregation.argument-setting-hint', + hiddenDefaultValue: true, + hiddenEntityKeyTypes: true, + }; + + this.isScript = false; + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 7e538a8dc7..b65ae4dc2f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -85,6 +85,13 @@ [tenantId]="data.tenantId" > } + @case (CalculatedFieldType.ENTITY_AGGREGATION) { + + } @default { +
+
+
+ {{ 'calculated-fields.arguments' | translate }} +
+
+ {{ 'calculated-fields.entity-aggregation.argument-hint' | translate }} +
+ +
+
+
+ {{ 'calculated-fields.metrics.metrics' | translate }} +
+ +
+
+
+ {{ 'calculated-fields.entity-aggregation.aggregation-interval' | translate }} +
+ +
+ + calculated-fields.aggregate-interval-type + + + {{ AggIntervalTypeTranslations.get(type) | translate }} + + + + + +
+ @if (entityAggregationConfiguration.get('interval.type').value === AggIntervalType.CUSTOM) { + + + } +
+ +
+ {{ 'calculated-fields.entity-aggregation.apply-offset' | translate }} +
+ + @if (entityAggregationConfiguration.get('interval.allowOffsetMillis').value) { + + + } +
+ +
+ +
+ {{ 'calculated-fields.entity-aggregation.wait-delay' | translate }} +
+ + @if (entityAggregationConfiguration.get('allowWatermark').value) { + + + + + + + } +
+
+ +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts new file mode 100644 index 0000000000..9f48187b55 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts @@ -0,0 +1,205 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { EntityId } from '@shared/models/id/entity-id'; +import { + AggInterval, + AggIntervalType, + AggIntervalTypeTranslations, + CalculatedFieldEntityAggregationConfiguration, + CalculatedFieldOutput, + CalculatedFieldType, + notEmptyObjectValidator, + OutputType +} from '@shared/models/calculated-field.models'; +import { map } from 'rxjs/operators'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { HOUR, MINUTE, SECOND } from '@shared/models/time/time.models'; +import { isDefinedAndNotNull } from '@core/utils'; + +interface CalculatedFieldEntityAggregationConfigurationValue extends CalculatedFieldEntityAggregationConfiguration { + interval: AggInterval & {allowOffsetMillis?: boolean}; + allowWatermark: boolean; +} + +@Component({ + selector: 'tb-entity-aggregation-component', + templateUrl: './entity-aggregation-component.component.html', + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => EntityAggregationComponentComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => EntityAggregationComponentComponent), + multi: true + } + ], +}) +export class EntityAggregationComponentComponent implements ControlValueAccessor, Validator { + + @Input({required: true}) + entityId: EntityId; + + @Input({required: true}) + tenantId: string; + + @Input({required: true}) + entityName: string; + + + entityAggregationConfiguration = this.fb.group({ + arguments: this.fb.control({}, notEmptyObjectValidator()), + metrics: this.fb.control({}, notEmptyObjectValidator()), + interval: this.fb.group({ + type: [AggIntervalType.HOUR], + tz: ['', Validators.required], + multiplier: [HOUR/SECOND, Validators.required], + allowOffsetMillis: [false], + offsetMillis: [MINUTE/SECOND, Validators.required], + }), + allowWatermark: [false], + watermark: this.fb.group({ + duration: [6 * MINUTE / SECOND, Validators.required], + checkInterval: [MINUTE / SECOND, Validators.required], + }), + output: this.fb.control({ + type: OutputType.Timeseries, + }), + }); + + arguments$ = this.entityAggregationConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => Object.keys(argumentsObj)) + ); + + AggIntervalType = AggIntervalType; + AggIntervalTypes = Object.values(AggIntervalType) as AggIntervalType[]; + AggIntervalTypeTranslations = AggIntervalTypeTranslations; + + private propagateChange: (config: CalculatedFieldEntityAggregationConfiguration) => void = () => { }; + + constructor(private fb: FormBuilder) { + + this.entityAggregationConfiguration.get('interval.type').valueChanges.pipe( + takeUntilDestroyed() + ).subscribe((type: AggIntervalType) => { + this.checkAggIntervalType(type); + }); + + this.entityAggregationConfiguration.get('interval.allowOffsetMillis').valueChanges.pipe( + takeUntilDestroyed() + ).subscribe((allow: boolean) => { + this.checkIntervalDuration(allow); + }); + + this.entityAggregationConfiguration.get('allowWatermark').valueChanges.pipe( + takeUntilDestroyed() + ).subscribe((allow: boolean) => { + this.checkWatermark(allow); + }); + + this.entityAggregationConfiguration.valueChanges.pipe( + takeUntilDestroyed() + ).subscribe((value: CalculatedFieldEntityAggregationConfigurationValue) => { + this.updatedModel(value); + }); + } + + validate(): ValidationErrors | null { + return this.entityAggregationConfiguration.valid || this.entityAggregationConfiguration.disabled ? null : {invalidPropagateConfig: false}; + } + + writeValue(value: CalculatedFieldEntityAggregationConfiguration): void { + const data: CalculatedFieldEntityAggregationConfigurationValue = { + ...value, + allowWatermark: isDefinedAndNotNull(value.watermark), + interval: {...value.interval, allowOffsetMillis: isDefinedAndNotNull(value?.interval?.offsetMillis)} + } + this.entityAggregationConfiguration.patchValue(data, {emitEvent: false}); + this.checkAggIntervalType(this.entityAggregationConfiguration.get('interval.type').value); + this.checkIntervalDuration(this.entityAggregationConfiguration.get('interval.allowOffsetMillis').value); + this.checkWatermark(this.entityAggregationConfiguration.get('allowWatermark').value); + setTimeout(() => { + this.entityAggregationConfiguration.get('arguments').updateValueAndValidity({onlySelf: true}); + }); + } + + registerOnChange(fn: (config: CalculatedFieldEntityAggregationConfiguration) => void): void { + this.propagateChange = fn; + } + + registerOnTouched(_: any): void { } + + setDisabledState(isDisabled: boolean): void { + if (isDisabled) { + this.entityAggregationConfiguration.disable({emitEvent: false}); + } else { + this.entityAggregationConfiguration.enable({emitEvent: false}); + this.checkAggIntervalType(this.entityAggregationConfiguration.get('interval.type').value); + this.checkIntervalDuration(this.entityAggregationConfiguration.get('interval.allowOffsetMillis').value); + this.checkWatermark(this.entityAggregationConfiguration.get('allowWatermark').value); + } + } + + private updatedModel(value: CalculatedFieldEntityAggregationConfigurationValue): void { + value.type = CalculatedFieldType.ENTITY_AGGREGATION; + if (!value.interval.allowOffsetMillis) { + delete value.interval.offsetMillis; + } + delete value.interval.offsetMillis; + if (!value.allowWatermark) { + delete value.watermark; + } + delete value.allowWatermark; + this.propagateChange(value); + } + + private checkAggIntervalType(type: AggIntervalType) { + if (type === AggIntervalType.CUSTOM) { + this.entityAggregationConfiguration.get('interval.multiplier').enable({emitEvent: false}); + } else { + this.entityAggregationConfiguration.get('interval.multiplier').disable({emitEvent: false}); + } + } + + private checkIntervalDuration(allow: boolean) { + if (allow) { + this.entityAggregationConfiguration.get('interval.offsetMillis').enable({emitEvent: false}); + } else { + this.entityAggregationConfiguration.get('interval.offsetMillis').disable({emitEvent: false}); + } + } + + private checkWatermark(allow: boolean) { + if (allow) { + this.entityAggregationConfiguration.get('watermark').enable({emitEvent: false}); + } else { + this.entityAggregationConfiguration.get('watermark').disable({emitEvent: false}); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.module.ts new file mode 100644 index 0000000000..44bbbc7237 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.module.ts @@ -0,0 +1,49 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { SharedModule } from '@shared/shared.module'; +import { + CalculatedFieldOutputModule +} from '@home/components/calculated-fields/components/output/calculated-field-output.module'; +import { + CalculatedFieldArgumentsTableModule +} from '@home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module'; +import { + EntityAggregationComponentComponent +} from '@home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component'; +import { + CalculatedFieldMetricsTableModule +} from '@home/components/calculated-fields/components/metrics/calculated-field-metrics-table.module'; + +@NgModule({ + imports: [ + CommonModule, + SharedModule, + CalculatedFieldOutputModule, + CalculatedFieldArgumentsTableModule, + CalculatedFieldMetricsTableModule, + ], + declarations: [ + EntityAggregationComponentComponent, + ], + exports: [ + EntityAggregationComponentComponent, + ] +}) +export class EntityAggregationComponentModule { +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-panel.component.html similarity index 73% rename from ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html rename to ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-panel.component.html index 8b29689b7f..d57a487dbd 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-panel.component.html @@ -77,48 +77,52 @@
-
- - - - -
- {{ 'calculated-fields.metrics.filter' | translate }} + @if (!simpleMode) { +
+ + + + +
+ {{ 'calculated-fields.metrics.filter' | translate }} +
+ + + + + +
{{ 'api-usage.tbel' | translate }}
- - - - - -
{{ 'api-usage.tbel' | translate }} -
- - - -
- -
-
{{ 'calculated-fields.metrics.value-source' | translate }}
- - - @for (inputType of AggInputTypes; track inputType) { - {{ AggInputTypeTranslations.get(inputType) | translate }} - } - - + + +
+ } + + @if (!simpleMode) { +
+
{{ 'calculated-fields.metrics.value-source' | translate }}
+ + + @for (inputType of AggInputTypes; track inputType) { + {{ AggInputTypeTranslations.get(inputType) | translate }} + } + + +
+ } @if (this.metricForm.get('input.type').value === AggInputType.key) {
{{ 'calculated-fields.argument-name' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-panel.component.ts similarity index 99% rename from ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.ts rename to ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-panel.component.ts index d21db8ece9..b06a49d60d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-panel.component.ts @@ -46,6 +46,7 @@ export class CalculatedFieldMetricsPanelComponent implements OnInit { @Input() metric: CalculatedFieldAggMetricValue; @Input() usedNames: string[]; @Input() arguments: Array; + @Input() simpleMode: boolean; @Input() editorCompleter: TbEditorCompleter; @Input() highlightRules: AceHighlightRules; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-table.component.html similarity index 91% rename from ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.html rename to ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-table.component.html index 4e05a0bbd5..67cfdfc122 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-table.component.html @@ -20,10 +20,16 @@ - +
{{ 'calculated-fields.metrics.metric-name' | translate }}
 - +
{{ metric.name }}
- + {{ 'calculated-fields.metrics.aggregation' | translate }} - +
{{ AggFunctionTranslations.get(metric.function) | translate }}
 diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-table.component.ts similarity index 94% rename from ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.ts rename to ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-table.component.ts index 3689e016b2..22adf9a801 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-table.component.ts @@ -16,11 +16,13 @@ import { AfterViewInit, + booleanAttribute, ChangeDetectorRef, Component, DestroyRef, forwardRef, Input, + OnInit, Renderer2, ViewChild, ViewContainerRef, @@ -51,7 +53,7 @@ import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { CalculatedFieldMetricsPanelComponent -} from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component'; +} from '@home/components/calculated-fields/components/metrics/calculated-field-metrics-panel.component'; import { TbEditorCompleter } from '@shared/models/ace/completion.models'; import { AceHighlightRules } from '@shared/models/ace/ace.models'; @@ -72,11 +74,12 @@ import { AceHighlightRules } from '@shared/models/ace/ace.models'; } ], }) -export class CalculatedFieldMetricsTableComponent implements ControlValueAccessor, Validator, AfterViewInit { +export class CalculatedFieldMetricsTableComponent implements OnInit, ControlValueAccessor, Validator, AfterViewInit { @Input() arguments: Array; @Input() editorCompleter: TbEditorCompleter; @Input() highlightRules: AceHighlightRules; + @Input({transform: booleanAttribute}) simpleMode: boolean = false; @ViewChild(MatSort, { static: true }) sort: MatSort; @@ -85,7 +88,7 @@ export class CalculatedFieldMetricsTableComponent implements ControlValueAccesso sortOrder = { direction: 'asc' as SortDirection, property: '' }; dataSource = new CalculatedFieldMetricsDatasource(); - displayColumns = ['name', 'function', 'filter', 'valueSource', 'actions'] + displayColumns = ['name', 'function', 'filter', 'valueSource', 'actions']; readonly AggFunctionTranslations = AggFunctionTranslations; readonly AggInputTypeTranslations = AggInputTypeTranslations; @@ -109,6 +112,12 @@ export class CalculatedFieldMetricsTableComponent implements ControlValueAccesso }); } + ngOnInit() { + if (this.simpleMode) { + this.displayColumns = ['name', 'function', 'actions']; + } + } + ngAfterViewInit(): void { this.sort.sortChange.asObservable().pipe( takeUntilDestroyed(this.destroyRef) @@ -155,7 +164,8 @@ export class CalculatedFieldMetricsTableComponent implements ControlValueAccesso usedNames: this.metricsFormArray.value.map(({ name }) => name).filter(name => name !== metric.name), arguments: this.arguments, editorCompleter: this.editorCompleter, - highlightRules: this.highlightRules + highlightRules: this.highlightRules, + simpleMode: this.simpleMode, }; this.popoverComponent = this.popoverService.displayPopover({ trigger, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-table.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-table.module.ts new file mode 100644 index 0000000000..10d391a0a4 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-table.module.ts @@ -0,0 +1,27 @@ +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { SharedModule } from '@shared/shared.module'; +import { + CalculatedFieldMetricsTableComponent +} from '@home/components/calculated-fields/components/metrics/calculated-field-metrics-table.component'; +import { + CalculatedFieldMetricsPanelComponent +} from '@home/components/calculated-fields/components/metrics/calculated-field-metrics-panel.component'; + + +@NgModule({ + imports: [ + CommonModule, + SharedModule, + ], + declarations: [ + CalculatedFieldMetricsTableComponent, + CalculatedFieldMetricsPanelComponent + ], + exports: [ + CalculatedFieldMetricsTableComponent + ] +}) +export class CalculatedFieldMetricsTableModule { + +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html index 4d8515ac7e..c33d033f7a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html @@ -17,7 +17,7 @@ -->
{{ 'calculated-fields.output' | translate }}
-
+
{{ 'calculated-fields.output-type' | translate }} @@ -44,7 +44,7 @@
@if (simpleMode) { @if (hiddenName) { -
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts index 2a9215b7cf..94224006a8 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts @@ -63,6 +63,13 @@ export class CalculatedFieldOutputComponent implements ControlValueAccessor, Val @coerceBoolean() hiddenName = false; + @Input() + @coerceBoolean() + disableType = false; + + @Input() + containerInputClass: string | string[] | Record = 'flex flex-col gap-3'; + @Input({required: true}) entityId: EntityId; @@ -141,6 +148,9 @@ export class CalculatedFieldOutputComponent implements ControlValueAccessor, Val if (this.simpleMode && 'name' in value) { value.name = value.name?.trim() ?? ''; } + if (this.disableType) { + value.type = this.outputForm.get('type').value; + } this.propagateChange(value); } @@ -163,5 +173,8 @@ export class CalculatedFieldOutputComponent implements ControlValueAccessor, Val } else { this.outputForm.get('decimalsByDefault').disable({emitEvent: false}); } + if (this.disableType) { + this.outputForm.get('type').disable({emitEvent: false}); + } } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html index 914226b777..8a0361ee58 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html @@ -60,6 +60,7 @@ [editorCompleter]="argumentsEditorCompleter$ | async" >
- @if (labelText && !inlineField) { @@ -36,13 +36,13 @@ warning } - + {{ hintText }} {{ hasError }} diff --git a/ui-ngx/src/app/shared/components/time-unit-input.component.ts b/ui-ngx/src/app/shared/components/time-unit-input.component.ts index 678d9e9f00..bcfdabd574 100644 --- a/ui-ngx/src/app/shared/components/time-unit-input.component.ts +++ b/ui-ngx/src/app/shared/components/time-unit-input.component.ts @@ -55,6 +55,9 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, @Input() labelText: string; + @Input() + hintText: string; + @Input() @coerceBoolean() required: boolean; @@ -86,6 +89,10 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, @coerceBoolean() inlineField: boolean; + @Input() + @coerceBoolean() + sameWidthInputs: boolean = false; + timeUnits = Object.values(TimeUnit).filter(item => item !== TimeUnit.MILLISECONDS) as TimeUnit[]; timeUnitTranslations = timeUnitTranslations; diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 0ef454ae1f..c792c2d06f 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -67,7 +67,8 @@ export enum CalculatedFieldType { SCRIPT = 'SCRIPT', GEOFENCING = 'GEOFENCING', PROPAGATION = 'PROPAGATION', - RELATED_ENTITIES_AGGREGATION = 'RELATED_ENTITIES_AGGREGATION' + RELATED_ENTITIES_AGGREGATION = 'RELATED_ENTITIES_AGGREGATION', + ENTITY_AGGREGATION = 'ENTITY_AGGREGATION', } export const CalculatedFieldTypeTranslations = new Map( @@ -77,6 +78,7 @@ export const CalculatedFieldTypeTranslations = new Map; } +export interface CalculatedFieldEntityAggregationConfiguration { + type: CalculatedFieldType.ENTITY_AGGREGATION; + arguments: Record; + metrics: Record; + interval: AggInterval; + watermark?: WatermarkConfig; + output: Omit; +} + +export interface WatermarkConfig { + duration?: number; + checkInterval?: number; +} + interface BasePropagationConfiguration { type: CalculatedFieldType.PROPAGATION; relation: RelationPathLevel; @@ -270,6 +287,35 @@ export const AggFunctionTranslations = new Map([ [AggFunction.COUNT_UNIQUE, 'calculated-fields.metrics.aggregation-type.count-unique'], ]) +export enum AggIntervalType { + HOUR = 'HOUR', + DAY = 'DAY', + WEEK = 'WEEK', + WEEK_SUN_SAT = 'WEEK_SUN_SAT', + MONTH = 'MONTH', + YEAR = 'YEAR', + CUSTOM = 'CUSTOM' +} + +export const AggIntervalTypeTranslations = new Map( + [ + [AggIntervalType.HOUR, 'calculated-fields.aggregate-period.hour'], + [AggIntervalType.DAY, 'calculated-fields.aggregate-period.day'], + [AggIntervalType.WEEK, 'calculated-fields.aggregate-period.week'], + [AggIntervalType.WEEK_SUN_SAT, 'calculated-fields.aggregate-period.week-sun-sat'], + [AggIntervalType.MONTH, 'calculated-fields.aggregate-period.month'], + [AggIntervalType.YEAR, 'calculated-fields.aggregate-period.year'], + [AggIntervalType.CUSTOM, 'calculated-fields.aggregate-period.custom'] + ] +); + +export interface AggInterval { + type: AggIntervalType; + tz: string; + offsetMillis?: number + multiplier?: number +} + export interface CalculatedFieldAggMetric { function: AggFunction; filter?: string; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index ab5fd6a3d0..58b100490d 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1057,7 +1057,8 @@ "script": "Script", "geofencing" : "Geofencing", "propagation": "Propagation", - "related-entities-aggregation": "Related entities aggregation" + "related-entities-aggregation": "Related entities aggregation", + "entity-aggregation": "Entity aggregation" }, "arguments": "Arguments", "decimals-by-default": "Decimals by default", @@ -1191,6 +1192,37 @@ "filter": "Filter", "filter-hint": "Enables filtering of entities during aggregation. The filter function must return a boolean value and can use all configured arguments." }, + "aggregate-interval-type": "Aggregate interval type", + "aggregate-interval-value": "Aggregate interval value", + "aggregate-interval-value-required": "Aggregate interval value is required", + "aggregate-period": { + "hour": "Hour", + "day": "Day", + "week": "Week (Mon - Sun)", + "week-sun-sat": "Week (Sun - Sat)", + "month": "Month", + "year": "Year", + "custom": "Custom" + }, + "entity-aggregation": { + "argument-hint": "Data will be fetched from selected entity", + "argument-setting-hint": "Latest telemetry is the only available argument type for this calculated field", + "aggregation-interval": "Aggregation interval", + "aggregation-interval-hint": "Defines how often to perform aggregation. Example: every 1 hour aggregates data at 00:00, 01:00, 02:00, etc.", + "apply-offset": "Apply offset to aggregation interval", + "apply-offset-hint": "Defines how much to shift the start of each aggregation period (e.g., +10 minutes - 00:10, 01:10).", + "offset-value": "Offset value", + "offset-value-required": "Offset value is required", + "wait-delay": "Wait for delayed telemetry", + "wait-delay-hint": "Waits for delayed telemetry after the interval ends.", + "wait-duration": "Duration", + "wait-duration-required": "Duration is required", + "wait-duration-hint": "Defines how long to wait for delayed data after the interval ends.", + "check-interval": "Check for telemetry every", + "check-interval-required": "Check for telemetry every is required", + "check-interval-max": "Check interval need be less than the duration", + "check-interval-hint": "Defines how often to recheck for late telemetry during the watermark period." + }, "hint": { "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with time series rolling type.", "arguments-propagate-arguments-with-rolling": "'Time series rolling' type is incompatible with 'Arguments only' propagation.", From 8087ca16efe595da069a7325112f70b85016b3ca Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 3 Nov 2025 08:36:33 +0200 Subject: [PATCH 10/40] added default value for metric --- ...AbstractCalculatedFieldProcessingService.java | 16 ++++++++++------ .../utils/CalculatedFieldArgumentUtils.java | 11 +++++++---- application/src/main/resources/thingsboard.yml | 2 ++ .../cf/configuration/aggregation/AggMetric.java | 1 + .../calculated-field-metrics-table.module.ts | 16 ++++++++++++++++ 5 files changed, 36 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 1b13bccda1..bae987d53c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -23,6 +23,7 @@ import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; @@ -53,7 +54,6 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntry; -import org.thingsboard.server.utils.CalculatedFieldArgumentUtils; import java.util.Collections; import java.util.HashMap; @@ -71,6 +71,7 @@ import static org.thingsboard.server.common.data.cf.configuration.geofencing.Ent import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LONGITUDE_ARGUMENT_KEY; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultAttributeEntry; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultKvEntry; +import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformAggMetricArgument; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformAggregationArgument; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformSingleValueArgument; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformTsRollingArgument; @@ -87,6 +88,9 @@ public abstract class AbstractCalculatedFieldProcessingService { protected ListeningExecutorService calculatedFieldCallbackExecutor; + @Value("${actors.calculated_fields.max_datapoints_limit}") + private int aggArgumentMaxDatapointsLimit; + @PostConstruct public void init() { calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( @@ -302,21 +306,21 @@ public abstract class AbstractCalculatedFieldProcessingService { AggFunction function = metric.getFunction(); long intervalMs = interval.getEndTs() - interval.getStartTs(); BaseReadTsKvQuery query = new BaseReadTsKvQuery(argKey, interval.getStartTs(), interval.getEndTs(), intervalMs, 1, Aggregation.valueOf(function.name())); - ListenableFuture argumentEntryFut = fetchTimeSeriesInternal(tenantId, entityId, query, CalculatedFieldArgumentUtils::transformAggMetricArgument); + ListenableFuture argumentEntryFut = fetchTimeSeriesInternal(tenantId, entityId, query, timeSeries -> transformAggMetricArgument(timeSeries, argKey, metric)); return resolveArgumentValue(argKey, argumentEntryFut); } private ListenableFuture fetchTimeSeries(TenantId tenantId, EntityId entityId, Argument argument, AggInterval interval, long queryEndTs) { long startInterval = interval.getCurrentIntervalStartTs(); long intervalEndTs = interval.getCurrentIntervalEndTs(); - ReadTsKvQuery query = buildTimeSeriesQuery(tenantId, argument, startInterval, queryEndTs); + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startInterval, queryEndTs, 0, aggArgumentMaxDatapointsLimit, Aggregation.NONE); return fetchTimeSeriesInternal(tenantId, entityId, query, timeSeries -> transformAggregationArgument(timeSeries, startInterval, intervalEndTs)); } private ListenableFuture fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument, long queryEndTs) { long argTimeWindow = argument.getTimeWindow() == 0 ? queryEndTs : argument.getTimeWindow(); long startInterval = queryEndTs - argTimeWindow; - ReadTsKvQuery query = buildTimeSeriesQuery(tenantId, argument, startInterval, queryEndTs); + ReadTsKvQuery query = buildTsRollingQuery(tenantId, argument, startInterval, queryEndTs); return fetchTimeSeriesInternal(tenantId, entityId, query, tsRolling -> transformTsRollingArgument(tsRolling, query.getLimit(), argTimeWindow)); } @@ -352,10 +356,10 @@ public abstract class AbstractCalculatedFieldProcessingService { }, calculatedFieldCallbackExecutor); } - private ReadTsKvQuery buildTimeSeriesQuery(TenantId tenantId, Argument argument, long startTs, long endTs) { + private ReadTsKvQuery buildTsRollingQuery(TenantId tenantId, Argument argument, long startTs, long endTs) { long maxDataPoints = apiLimitService.getLimit( tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); - int argumentLimit = argument.getLimit() == null ? 500000 : argument.getLimit(); + int argumentLimit = argument.getLimit(); int limit = argumentLimit == 0 || argumentLimit > maxDataPoints ? (int) maxDataPoints : argumentLimit; return new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, endTs, 0, limit, Aggregation.NONE); } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index c6c64782a9..676903e239 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors; import org.apache.commons.lang3.math.NumberUtils; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; @@ -67,9 +68,9 @@ public class CalculatedFieldArgumentUtils { return ArgumentEntry.createTsRollingArgument(tsRolling, limit, argTimeWindow); } - public static ArgumentEntry transformAggMetricArgument(List timeSeries) { + public static ArgumentEntry transformAggMetricArgument(List timeSeries, String argKey, AggMetric aggMetric) { if (timeSeries == null || timeSeries.isEmpty()) { - return new SingleValueArgumentEntry(); + return ArgumentEntry.createSingleValueArgument(createDefaultKvEntry(argKey, aggMetric.getDefaultValue())); } return ArgumentEntry.createSingleValueArgument(timeSeries.get(0)); } @@ -86,8 +87,10 @@ public class CalculatedFieldArgumentUtils { } public static KvEntry createDefaultKvEntry(Argument argument) { - String key = argument.getRefEntityKey().getKey(); - String defaultValue = argument.getDefaultValue(); + return createDefaultKvEntry(argument.getRefEntityKey().getKey(), argument.getDefaultValue()); + } + + public static KvEntry createDefaultKvEntry(String key, String defaultValue) { if (StringUtils.isBlank(defaultValue)) { return new StringDataEntry(key, null); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 88f2f017d3..b29970417a 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -531,6 +531,8 @@ actors: calculation_timeout: "${ACTORS_CALCULATION_TIMEOUT_SEC:5}" # Interval in seconds to re-evaluate calculated fields that have a time schedule. 2 minutes by default. check_interval: "${ACTORS_CALCULATED_FIELDS_CHECK_INTERVAL_SEC:120}" + # Maximum allowed datapoints fetched by aggregation calculated fields + max_datapoints_limit: "${CF_AGG_MAX_DATAPOINTS_LIMIT:50000}" debug: settings: diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java index ebd612b1e0..8bf2818bd2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java @@ -27,5 +27,6 @@ public class AggMetric { private AggFunction function; private String filter; private AggInput input; + private String defaultValue; } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-table.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-table.module.ts index 10d391a0a4..14a6a3963f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-table.module.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-table.module.ts @@ -1,3 +1,19 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + import { NgModule } from '@angular/core'; import { CommonModule } from '@angular/common'; import { SharedModule } from '@shared/shared.module'; From f42c62ac7481a80cfda9acdc5c453d7f62391f15 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 3 Nov 2025 09:19:17 +0200 Subject: [PATCH 11/40] added test --- .../single/AggIntervalEntryStatus.java | 8 + ...EntityAggregationCalculatedFieldState.java | 23 ++- .../EntityAggregationCalculatedFieldTest.java | 176 ++++++++++++++++++ 3 files changed, 200 insertions(+), 7 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/cf/EntityAggregationCalculatedFieldTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntryStatus.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntryStatus.java index fa9bbd5a54..fd2a899503 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntryStatus.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntryStatus.java @@ -45,4 +45,12 @@ public class AggIntervalEntryStatus { return false; } + public boolean intervalPassed(long checkInterval) { + boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - checkInterval; + if (intervalPassed) { + lastMetricsEvalTs = System.currentTimeMillis(); + } + return intervalPassed; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java index 6885aa8b42..5510d63f24 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java @@ -41,6 +41,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultKvEntry; + public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldState { private AggInterval interval; @@ -58,8 +60,8 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt } public void scheduleReevaluation() { - fillMissingIntervals(interval.getCurrentIntervalEndTs(), intervalDuration); prepareIntervals(); + fillMissingIntervals(interval.getCurrentIntervalEndTs(), intervalDuration); long now = System.currentTimeMillis(); intervals.forEach((intervalEntry, argumentIntervalStatuses) -> { if (intervalEntry.belongsToInterval(now)) { @@ -115,8 +117,8 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception { - prepareIntervals(); createIntervalIfNotExist(); + prepareIntervals(); long now = System.currentTimeMillis(); Map> results = new HashMap<>(); @@ -163,8 +165,10 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt } arguments.forEach((argName, argumentEntry) -> { var entityAggEntry = (EntityAggregationArgumentEntry) argumentEntry; - entityAggEntry.getAggIntervals().put(currentInterval, new AggIntervalEntryStatus()); - intervals.computeIfAbsent(currentInterval, i -> new HashMap<>()).put(argName, new AggIntervalEntryStatus()); + if (!entityAggEntry.getAggIntervals().containsKey(currentInterval)) { + entityAggEntry.getAggIntervals().put(currentInterval, new AggIntervalEntryStatus()); + intervals.computeIfAbsent(currentInterval, i -> new HashMap<>()).put(argName, new AggIntervalEntryStatus()); + } }); ctx.scheduleReevaluation(interval.getDelayUntilIntervalEnd(), actorCtx); } @@ -190,7 +194,7 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt Map> results) { args.forEach((argName, argEntryIntervalStatus) -> { if (argEntryIntervalStatus.getLastArgsRefreshTs() > argEntryIntervalStatus.getLastMetricsEvalTs()) { - processMetric(intervalEntry, argName, results); + processMetric(intervalEntry, argName, false, results); } }); } @@ -200,20 +204,25 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt Map> results) { args.forEach((argName, argEntryIntervalStatus) -> { if (argEntryIntervalStatus.shouldRecalculate(checkInterval)) { - processMetric(intervalEntry, argName, results); + processMetric(intervalEntry, argName, false, results); ctx.scheduleReevaluation(checkInterval, actorCtx); + } else if (argEntryIntervalStatus.intervalPassed(checkInterval)) { + processMetric(intervalEntry, argName, true, results); } }); } private void processMetric(AggIntervalEntry intervalEntry, String argName, + boolean useDefault, Map> results) { String metricName = findMetricName(argName); if (metricName != null) { AggMetric metric = metrics.get(metricName); String argKey = ctx.getArguments().get(argName).getRefEntityKey().getKey(); - ArgumentEntry metricEntry = cfProcessingService.fetchMetricDuringInterval(ctx.getTenantId(), entityId, argKey, metric, intervalEntry); + ArgumentEntry metricEntry = useDefault + ? ArgumentEntry.createSingleValueArgument(createDefaultKvEntry(argKey, metric.getDefaultValue())) + : cfProcessingService.fetchMetricDuringInterval(ctx.getTenantId(), entityId, argKey, metric, intervalEntry); if (!metricEntry.isEmpty()) { results.computeIfAbsent(intervalEntry, i -> new HashMap<>()).put(metricName, metricEntry); } diff --git a/application/src/test/java/org/thingsboard/server/cf/EntityAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/EntityAggregationCalculatedFieldTest.java new file mode 100644 index 0000000000..2509263dce --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/cf/EntityAggregationCalculatedFieldTest.java @@ -0,0 +1,176 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cf; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.test.annotation.DirtiesContext; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.EntityAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.AggInterval; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.AggIntervalType; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.CustomInterval; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.Watermark; +import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.thingsboard.server.cf.CalculatedFieldIntegrationTest.POLL_INTERVAL; + +@DaoSqlTest +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) +public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest { + + private Tenant savedTenant; + + @Before + public void beforeEach() throws Exception { + loginSysAdmin(); + + updateDefaultTenantProfileConfig(tenantProfileConfig -> { + tenantProfileConfig.setMinAllowedDeduplicationIntervalInSecForCF(1); + }); + + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + savedTenant = saveTenant(tenant); + assertThat(savedTenant).isNotNull(); + + User tenantAdmin = new User(); + tenantAdmin.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin.setTenantId(savedTenant.getId()); + tenantAdmin.setEmail("tenant@thingsboard.org"); + tenantAdmin.setFirstName("John"); + tenantAdmin.setLastName("Doe"); + + createUserAndLogin(tenantAdmin, "testPassword"); + } + + @After + public void afterTest() throws Exception { + loginSysAdmin(); + + deleteTenant(savedTenant.getId()); + } + + @Test + public void testCreateCf_checkAggregation() throws Exception { + Device device = createDevice("Device", "1234567890111"); + + CustomInterval customInterval = new CustomInterval(1, AggIntervalType.MIN, 0, "Europe/Kyiv"); + long currentIntervalStartTs = customInterval.getCurrentIntervalStartTs(); + long currentIntervalEndTs = customInterval.getCurrentIntervalEndTs(); + + long tsBeforeInterval = currentIntervalStartTs - 1000L; + long tsInInterval_1 = currentIntervalStartTs + 1000L; + long tsInInterval_2 = currentIntervalStartTs + 500L; + long tsInInterval_3 = currentIntervalStartTs + 200L; + postTelemetry(device.getId(), String.format("{\"ts\": \"%s\", \"values\": {\"energy\":120}}", tsBeforeInterval)); + postTelemetry(device.getId(), String.format("{\"ts\": \"%s\", \"values\": {\"energy\":100}}", tsInInterval_1)); + postTelemetry(device.getId(), String.format("{\"ts\": \"%s\", \"values\": {\"energy\":180}}", tsInInterval_2)); + postTelemetry(device.getId(), String.format("{\"ts\": \"%s\", \"values\": {\"energy\":120}}", tsInInterval_3)); + + long interval = customInterval.getIntervalDurationMillis(); + CalculatedField totalConsumptionCF = createTotalConsumptionCF(device.getId(), customInterval); + + await().alias("create CF and perform aggregation after interval end") + .atMost(2 * interval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode result = getLatestTelemetry(device.getId(), "consumptionPerMin"); + assertThat(result).isNotNull(); + assertThat(result.get("consumptionPerMin").get(0).get("value").asText()).isEqualTo("400"); + }); + } + + private CalculatedField createTotalConsumptionCF(EntityId entityId, AggInterval aggInterval) { + Map arguments = new HashMap<>(); + Argument argument = new Argument(); + argument.setRefEntityKey(new ReferencedEntityKey("energy", ArgumentType.TS_LATEST, null)); + argument.setLimit(100); + arguments.put("en", argument); + + Map aggMetrics = new HashMap<>(); + + AggMetric consumptionPerMin = new AggMetric(); + consumptionPerMin.setFunction(AggFunction.SUM); + consumptionPerMin.setInput(new AggKeyInput("en")); + aggMetrics.put("consumptionPerMin", consumptionPerMin); + + Output output = new Output(); + output.setType(OutputType.TIME_SERIES); + output.setDecimalsByDefault(0); + + return createAggCf("Consumption per minute", entityId, + aggInterval, + new Watermark(TimeUnit.MINUTES.toMillis(1), TimeUnit.SECONDS.toMillis(10)), + arguments, + aggMetrics, + output); + } + + private CalculatedField createAggCf(String name, + EntityId entityId, + AggInterval aggInterval, + Watermark watermark, + Map inputs, + Map metrics, + Output output) { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setName(name); + calculatedField.setEntityId(entityId); + calculatedField.setType(CalculatedFieldType.ENTITY_AGGREGATION); + + EntityAggregationCalculatedFieldConfiguration configuration = new EntityAggregationCalculatedFieldConfiguration(); + + configuration.setArguments(inputs); + configuration.setMetrics(metrics); + configuration.setInterval(aggInterval); + configuration.setWatermark(watermark); + configuration.setOutput(output); + + calculatedField.setConfiguration(configuration); + calculatedField.setDebugSettings(DebugSettings.all()); + return saveCalculatedField(calculatedField); + } + + private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); + } + +} From 7000e7318d641a477f3a7d78ae03ff2be2f75357 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 3 Nov 2025 10:53:04 +0200 Subject: [PATCH 12/40] removed min agg interval type --- .../EntityAggregationCalculatedFieldTest.java | 3 +- .../single/interval/AggInterval.java | 1 - .../single/interval/AggIntervalType.java | 1 - .../single/interval/BaseAggInterval.java | 125 ++++++------------ .../single/interval/CustomInterval.java | 32 +++-- .../single/interval/MinInterval.java | 30 ----- 6 files changed, 67 insertions(+), 125 deletions(-) delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MinInterval.java diff --git a/application/src/test/java/org/thingsboard/server/cf/EntityAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/EntityAggregationCalculatedFieldTest.java index 2509263dce..5a73b026f0 100644 --- a/application/src/test/java/org/thingsboard/server/cf/EntityAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/EntityAggregationCalculatedFieldTest.java @@ -35,7 +35,6 @@ import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInp import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; import org.thingsboard.server.common.data.cf.configuration.aggregation.single.EntityAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.AggInterval; -import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.AggIntervalType; import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.CustomInterval; import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.Watermark; import org.thingsboard.server.common.data.debug.DebugSettings; @@ -92,7 +91,7 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest public void testCreateCf_checkAggregation() throws Exception { Device device = createDevice("Device", "1234567890111"); - CustomInterval customInterval = new CustomInterval(1, AggIntervalType.MIN, 0, "Europe/Kyiv"); + CustomInterval customInterval = new CustomInterval(60L, 0L, "Europe/Kyiv"); long currentIntervalStartTs = customInterval.getCurrentIntervalStartTs(); long currentIntervalEndTs = customInterval.getCurrentIntervalEndTs(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java index d923eb5d7a..d6d1aca6a8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java @@ -26,7 +26,6 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; property = "type" ) @JsonSubTypes({ - @JsonSubTypes.Type(value = MinInterval.class, name = "MIN"), @JsonSubTypes.Type(value = HourInterval.class, name = "HOUR"), @JsonSubTypes.Type(value = DayInterval.class, name = "DAY"), @JsonSubTypes.Type(value = WeekInterval.class, name = "WEEK"), diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java index 96bac69128..62185127ed 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java @@ -17,7 +17,6 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.i public enum AggIntervalType { - MIN, HOUR, DAY, WEEK, diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java index d8df2d9826..e64ec117fd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java @@ -15,11 +15,11 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.Data; import java.time.DayOfWeek; import java.time.Duration; -import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime; import java.time.ZoneId; @@ -28,127 +28,90 @@ import java.time.temporal.ChronoUnit; import java.time.temporal.TemporalAdjusters; @Data +@JsonInclude(JsonInclude.Include.NON_NULL) public abstract class BaseAggInterval implements AggInterval { protected String tz; - protected long offsetMillis; // delay millis since start of interval + protected Long offsetSec; // delay seconds since start of interval @Override public long getIntervalDurationMillis() { - return getIntervalDurationMillis(getType(), 1); - } - - public long getIntervalDurationMillis(AggIntervalType type, int multiplier) { - return switch (type) { - case MIN -> Duration.ofMinutes(multiplier).toMillis(); - case HOUR -> Duration.ofHours(multiplier).toMillis(); - case DAY -> Duration.ofDays(multiplier).toMillis(); - case WEEK, WEEK_SUN_SAT -> Duration.ofDays(7L * multiplier).toMillis(); - case MONTH -> Duration.ofDays(Math.round(30 * multiplier)).toMillis(); // average - case YEAR -> Duration.ofDays(Math.round(365 * multiplier)).toMillis(); - default -> throw new IllegalArgumentException("Unsupported type: " + type); + return switch (getType()) { + case HOUR -> Duration.ofHours(1).toMillis(); + case DAY -> Duration.ofDays(1).toMillis(); + case WEEK, WEEK_SUN_SAT -> Duration.ofDays(7L).toMillis(); + case MONTH -> Duration.ofDays(Math.round(30)).toMillis(); // average + case YEAR -> Duration.ofDays(Math.round(365)).toMillis(); + default -> throw new IllegalArgumentException("Unsupported type: " + getType()); }; } @Override public long getCurrentIntervalStartTs() { - return getCurrentIntervalStartTs(getType(), 1); - } - - protected long getCurrentIntervalStartTs(AggIntervalType type, int multiplier) { ZoneId zoneId = ZoneId.of(tz); ZonedDateTime now = ZonedDateTime.now(zoneId); - ZonedDateTime shiftedNow = now.minus(Duration.ofMillis(offsetMillis)); - ZonedDateTime alignedStart = getAlignedBoundary(type, multiplier, false, shiftedNow); - ZonedDateTime actualStart = alignedStart.plus(Duration.ofMillis(offsetMillis)); + ZonedDateTime shiftedNow = now.minusSeconds(offsetSec); + ZonedDateTime alignedStart = getAlignedBoundary(shiftedNow, false); + ZonedDateTime actualStart = alignedStart.plusSeconds(offsetSec); return actualStart.toInstant().toEpochMilli(); } @Override public long getCurrentIntervalEndTs() { - return getCurrentIntervalEndTs(getType(), 1); - } - - protected long getCurrentIntervalEndTs(AggIntervalType type, int multiplier) { ZoneId zoneId = ZoneId.of(tz); ZonedDateTime now = ZonedDateTime.now(zoneId); - ZonedDateTime shiftedNow = now.minus(Duration.ofMillis(offsetMillis)); - ZonedDateTime alignedEnd = getAlignedBoundary(type, multiplier, true, shiftedNow); - ZonedDateTime actualEnd = alignedEnd.plus(Duration.ofMillis(offsetMillis)); + ZonedDateTime shiftedNow = now.minusSeconds(offsetSec); + ZonedDateTime alignedEnd = getAlignedBoundary(shiftedNow, true); + ZonedDateTime actualEnd = alignedEnd.plusSeconds(offsetSec); return actualEnd.toInstant().toEpochMilli(); } @Override public long getDelayUntilIntervalEnd() { - return getDelayUntilIntervalEnd(getType(), 1); - } - - protected long getDelayUntilIntervalEnd(AggIntervalType type, int multiplier) { - long currentIntervalEndTs = getCurrentIntervalEndTs(type, multiplier); + long currentIntervalEndTs = getCurrentIntervalEndTs(); long now = System.currentTimeMillis(); return currentIntervalEndTs - now; } - protected ZonedDateTime getAlignedBoundary(AggIntervalType type, int multiplier, boolean next, ZonedDateTime reference) { - return switch (type) { - case MIN -> alignByMin(reference, multiplier, next); - case HOUR -> alignByHour(reference, multiplier, next); - case DAY -> alignByDay(reference, multiplier, next); - case WEEK -> alignByWeek(reference, multiplier, DayOfWeek.MONDAY, next); - case WEEK_SUN_SAT -> alignByWeek(reference, multiplier, DayOfWeek.SUNDAY, next); - case MONTH -> alignByMonth(reference, multiplier, next); - case YEAR -> alignByYear(reference, multiplier, next); - default -> throw new IllegalArgumentException("Unsupported type: " + type); + protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { + return switch (getType()) { + case HOUR -> alignByHours(reference, next); + case DAY -> alignByDays(reference, next); + case WEEK -> alignByWeeks(reference, DayOfWeek.MONDAY, next); + case WEEK_SUN_SAT -> alignByWeeks(reference, DayOfWeek.SUNDAY, next); + case MONTH -> alignByMonths(reference, next); + case YEAR -> alignByYears(reference, next); + default -> throw new IllegalArgumentException("Unsupported interval type: " + getType()); }; } - private ZonedDateTime alignByMin(ZonedDateTime now, int multiplier, boolean next) { - ZonedDateTime startOfHour = now.truncatedTo(ChronoUnit.HOURS); - long minsSinceHour = Duration.between(startOfHour, now).toMinutes(); - long aligned = (minsSinceHour / multiplier) * multiplier; - if (next) aligned += multiplier; - return startOfHour.plusMinutes(aligned); - } - - private ZonedDateTime alignByHour(ZonedDateTime now, int multiplier, boolean next) { - ZonedDateTime startOfDay = now.truncatedTo(ChronoUnit.DAYS); - long hoursSinceMidnight = Duration.between(startOfDay, now).toHours(); - long aligned = (hoursSinceMidnight / multiplier) * multiplier; - if (next) aligned += multiplier; - return startOfDay.plusHours(aligned); + private ZonedDateTime alignByHours(ZonedDateTime now, boolean next) { + ZonedDateTime base = now.truncatedTo(ChronoUnit.HOURS); + return next ? base.plusHours(1) : base; } - private ZonedDateTime alignByDay(ZonedDateTime now, int multiplier, boolean next) { - long daysSinceEpoch = now.toLocalDate().toEpochDay(); - long aligned = (daysSinceEpoch / multiplier) * multiplier; - if (next) aligned += multiplier; - long diff = aligned - daysSinceEpoch; - return now.truncatedTo(ChronoUnit.DAYS).plusDays(diff); + private ZonedDateTime alignByDays(ZonedDateTime now, boolean next) { + ZonedDateTime base = now.truncatedTo(ChronoUnit.DAYS); + return next ? base.plusDays(1) : base; } - private ZonedDateTime alignByWeek(ZonedDateTime now, int multiplier, DayOfWeek startOfWeekDay, boolean next) { - ZonedDateTime startOfWeek = now.with(TemporalAdjusters.previousOrSame(startOfWeekDay)) + private ZonedDateTime alignByWeeks(ZonedDateTime now, DayOfWeek startOfWeek, boolean next) { + ZonedDateTime startOfWeekDate = now.with(TemporalAdjusters.previousOrSame(startOfWeek)) .truncatedTo(ChronoUnit.DAYS); - long weeksSinceEpoch = ChronoUnit.WEEKS.between( - ZonedDateTime.ofInstant(Instant.EPOCH, now.getZone()), startOfWeek); - long aligned = (weeksSinceEpoch / multiplier) * multiplier; - if (next) aligned += multiplier; - return startOfWeek.plusWeeks(aligned - weeksSinceEpoch); + return next ? startOfWeekDate.plusWeeks(1) : startOfWeekDate; } - private ZonedDateTime alignByMonth(ZonedDateTime now, int multiplier, boolean next) { - ZonedDateTime startOfMonth = now.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS); - long monthsSinceEpoch = now.getYear() * 12L + now.getMonthValue() - 1; - long aligned = (monthsSinceEpoch / multiplier) * multiplier; - if (next) aligned += multiplier; - return startOfMonth.plusMonths(aligned - monthsSinceEpoch); + private ZonedDateTime alignByMonths(ZonedDateTime now, boolean next) { + ZonedDateTime base = now.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS); + return next ? base.plusMonths(1) : base; } - private ZonedDateTime alignByYear(ZonedDateTime now, int multiplier, boolean next) { - int year = now.getYear(); - int aligned = (year / multiplier) * multiplier; - if (next) aligned += multiplier; - return ZonedDateTime.of(LocalDate.of(aligned, 1, 1), LocalTime.MIDNIGHT, now.getZone()); + private ZonedDateTime alignByYears(ZonedDateTime now, boolean next) { + ZonedDateTime base = ZonedDateTime.of( + LocalDate.of(now.getYear(), 1, 1), + LocalTime.MIDNIGHT, + now.getZone()); + return next ? base.plusYears(1) : base; } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java index cc56b2aef3..3a07029bc8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java @@ -19,19 +19,22 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; +import java.time.Duration; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.concurrent.TimeUnit; + @EqualsAndHashCode(callSuper = true) @Data @NoArgsConstructor public class CustomInterval extends BaseAggInterval { - private int multiplier; // number of base units (e.g. 2 hours, 5 days) - private AggIntervalType internalIntervalType; + private Long durationSec; - public CustomInterval(int multiplier, AggIntervalType internalIntervalType, long offsetMillis, String tz) { + public CustomInterval(Long durationSec, Long offsetMillis, String tz) { this.tz = tz; - this.offsetMillis = offsetMillis; - this.multiplier = multiplier; - this.internalIntervalType = internalIntervalType; + this.offsetSec = offsetMillis; + this.durationSec = durationSec; } @Override @@ -41,22 +44,31 @@ public class CustomInterval extends BaseAggInterval { @Override public long getIntervalDurationMillis() { - return getIntervalDurationMillis(internalIntervalType, multiplier); + return Duration.ofSeconds(durationSec).toMillis(); } @Override public long getCurrentIntervalStartTs() { - return super.getCurrentIntervalStartTs(internalIntervalType, multiplier); + ZoneId zoneId = ZoneId.of(tz); + ZonedDateTime now = ZonedDateTime.now(zoneId); + ZonedDateTime shiftedNow = now.minusSeconds(offsetSec); + + long durationMillis = getIntervalDurationMillis(); + long shiftedNowMillis = shiftedNow.toInstant().toEpochMilli(); + long alignedStartMillis = (shiftedNowMillis / durationMillis) * durationMillis; + + long offsetMillis = TimeUnit.SECONDS.toMillis(offsetSec); + return alignedStartMillis + offsetMillis; } @Override public long getCurrentIntervalEndTs() { - return super.getCurrentIntervalEndTs(internalIntervalType, multiplier); + return getCurrentIntervalStartTs() + getIntervalDurationMillis(); } @Override public long getDelayUntilIntervalEnd() { - return super.getDelayUntilIntervalEnd(internalIntervalType, multiplier); + return getCurrentIntervalEndTs() - System.currentTimeMillis(); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MinInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MinInterval.java deleted file mode 100644 index 066bc230b8..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MinInterval.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; - -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -public class MinInterval extends BaseAggInterval { - - @Override - public AggIntervalType getType() { - return AggIntervalType.MIN; - } - -} From 3842b9af1b41c0d51ec28694ad35d4c0d6c5231d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 3 Nov 2025 10:59:39 +0200 Subject: [PATCH 13/40] added safe get --- .../single/interval/BaseAggInterval.java | 16 ++++++++++++---- .../single/interval/CustomInterval.java | 4 ++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java index e64ec117fd..08bea68d5b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import lombok.Data; @@ -34,6 +35,11 @@ public abstract class BaseAggInterval implements AggInterval { protected String tz; protected Long offsetSec; // delay seconds since start of interval + @JsonIgnore + protected long getOffsetSec() { + return offsetSec != null ? offsetSec : 0L; + } + @Override public long getIntervalDurationMillis() { return switch (getType()) { @@ -50,9 +56,10 @@ public abstract class BaseAggInterval implements AggInterval { public long getCurrentIntervalStartTs() { ZoneId zoneId = ZoneId.of(tz); ZonedDateTime now = ZonedDateTime.now(zoneId); - ZonedDateTime shiftedNow = now.minusSeconds(offsetSec); + long offset = getOffsetSec(); + ZonedDateTime shiftedNow = now.minusSeconds(offset); ZonedDateTime alignedStart = getAlignedBoundary(shiftedNow, false); - ZonedDateTime actualStart = alignedStart.plusSeconds(offsetSec); + ZonedDateTime actualStart = alignedStart.plusSeconds(offset); return actualStart.toInstant().toEpochMilli(); } @@ -60,9 +67,10 @@ public abstract class BaseAggInterval implements AggInterval { public long getCurrentIntervalEndTs() { ZoneId zoneId = ZoneId.of(tz); ZonedDateTime now = ZonedDateTime.now(zoneId); - ZonedDateTime shiftedNow = now.minusSeconds(offsetSec); + long offset = getOffsetSec(); + ZonedDateTime shiftedNow = now.minusSeconds(offset); ZonedDateTime alignedEnd = getAlignedBoundary(shiftedNow, true); - ZonedDateTime actualEnd = alignedEnd.plusSeconds(offsetSec); + ZonedDateTime actualEnd = alignedEnd.plusSeconds(offset); return actualEnd.toInstant().toEpochMilli(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java index 3a07029bc8..c8e3ee15a6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java @@ -51,13 +51,13 @@ public class CustomInterval extends BaseAggInterval { public long getCurrentIntervalStartTs() { ZoneId zoneId = ZoneId.of(tz); ZonedDateTime now = ZonedDateTime.now(zoneId); - ZonedDateTime shiftedNow = now.minusSeconds(offsetSec); + ZonedDateTime shiftedNow = now.minusSeconds(getOffsetSec()); long durationMillis = getIntervalDurationMillis(); long shiftedNowMillis = shiftedNow.toInstant().toEpochMilli(); long alignedStartMillis = (shiftedNowMillis / durationMillis) * durationMillis; - long offsetMillis = TimeUnit.SECONDS.toMillis(offsetSec); + long offsetMillis = TimeUnit.SECONDS.toMillis(getOffsetSec()); return alignedStartMillis + offsetMillis; } From 0a45184f4d7bc96752414abf398ae28b37b2b683 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Mon, 3 Nov 2025 12:20:55 +0200 Subject: [PATCH 14/40] UI: Calculated field refactor arguments name validators --- ...lculated-field-argument-panel.component.ts | 34 ++++++++----------- .../propagate-arguments-table.component.ts | 10 ++++-- ...-geofencing-zone-groups-panel.component.ts | 30 ++++++---------- .../shared/models/calculated-field.models.ts | 27 ++++++++++++++- 4 files changed, 59 insertions(+), 42 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts index e58145cf05..c61153d6b9 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts @@ -25,7 +25,7 @@ import { ViewChild } from '@angular/core'; import { TbPopoverComponent } from '@shared/components/popover.component'; -import { FormBuilder, FormControl, FormGroup, ValidatorFn, Validators } from '@angular/forms'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; import { charsWithNumRegex, oneSpaceInsideRegex } from '@shared/models/regex.constants'; import { ArgumentEntityType, @@ -34,7 +34,10 @@ import { ArgumentType, ArgumentTypeTranslations, CalculatedFieldArgumentValue, - getCalculatedFieldCurrentEntityFilter + FORBIDDEN_NAMES, + forbiddenNamesValidator, + getCalculatedFieldCurrentEntityFilter, + uniqueNameValidator } from '@shared/models/calculated-field.models'; import { debounceTime, distinctUntilChanged, filter } from 'rxjs/operators'; import { EntityType } from '@shared/models/entity-type.models'; @@ -74,6 +77,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI @Input() defaultValueRequired = false; @Input() hint: string; @Input() predefinedEntityFilter: EntityFilter; + @Input() forbiddenNames = FORBIDDEN_NAMES; @Input() argumentEntityTypes = Object.values(ArgumentEntityType).filter(value => value !== ArgumentEntityType.RelationQuery) as ArgumentEntityType[]; @ViewChild('entityAutocomplete') entityAutocomplete: EntityAutocompleteComponent; @@ -86,7 +90,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI readonly defaultLimit = Math.floor(this.maxDataPointsPerRollingArg / 10); argumentFormGroup = this.fb.group({ - argumentName: ['', [Validators.required, this.uniqNameRequired(), this.forbiddenArgumentNameValidator(), Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], + argumentName: ['', [Validators.required, Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], refEntityId: [null], refEntityKey: this.fb.group({ type: [ArgumentType.LatestTelemetry, [Validators.required]], @@ -141,6 +145,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI } ngOnInit(): void { + this.updatedFormValidators(); this.updatedArgumentType(); this.argumentFormGroup.patchValue(this.argument, {emitEvent: false}); this.currentEntityFilter = getCalculatedFieldCurrentEntityFilter(this.entityName, this.entityId); @@ -185,6 +190,12 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI this.popover.hide(); } + private updatedFormValidators(): void { + this.argumentFormGroup.get('argumentName').addValidators( + [uniqueNameValidator(this.usedArgumentNames), forbiddenNamesValidator(this.forbiddenNames)]); + this.argumentFormGroup.get('argumentName').updateValueAndValidity({emitEvent: false}); + } + private updatedArgumentType(): void { let argumentType = ArgumentEntityType.Current; if (this.argument.refEntityId?.entityType) { @@ -255,15 +266,6 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI }); } - private uniqNameRequired(): ValidatorFn { - return (control: FormControl) => { - const newName = control.value.trim().toLowerCase(); - const isDuplicate = this.usedArgumentNames?.some(name => name.toLowerCase() === newName); - - return isDuplicate ? { duplicateName: true } : null; - }; - } - private observeEntityKeyChanges(): void { this.argumentFormGroup.get('refEntityKey').get('type').valueChanges .pipe(takeUntilDestroyed()) @@ -297,14 +299,6 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI } } - private forbiddenArgumentNameValidator(): ValidatorFn { - return (control: FormControl) => { - const trimmedValue = control.value.trim().toLowerCase(); - const forbiddenArgumentNames = ['ctx', 'e', 'pi', 'propagationCtx']; - return forbiddenArgumentNames.includes(trimmedValue) ? { forbiddenName: true } : null; - }; - } - private updatedRefEntityIdState(type: ArgumentEntityType): void { const isEntityWithId = !!type && type !== ArgumentEntityType.Tenant && type !== ArgumentEntityType.Current; this.argumentFormGroup.get('refEntityId')[isEntityWithId ? 'enable' : 'disable'](); diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/propagate-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/propagate-arguments-table.component.ts index 04d1dbf91b..2e53376f77 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/propagate-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/propagate-arguments-table.component.ts @@ -31,7 +31,12 @@ import { AppState } from '@core/core.state'; import { CalculatedFieldArgumentsTableComponent } from '@home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.component'; -import { ArgumentEntityType, ArgumentType, CalculatedFieldArgumentValue } from '@shared/models/calculated-field.models'; +import { + ArgumentEntityType, + ArgumentType, + CalculatedFieldArgumentValue, + FORBIDDEN_NAMES +} from '@shared/models/calculated-field.models'; import { isDefined } from '@core/utils'; import { NULL_UUID } from '@shared/models/id/has-uuid'; @@ -88,7 +93,8 @@ export class PropagateArgumentsTableComponent extends CalculatedFieldArgumentsTa this.displayColumns = ['name', 'type', 'key', 'actions']; this.panelAdditionalCtx = { argumentEntityTypes: [ArgumentEntityType.Current], - isOutputKey: true + isOutputKey: true, + forbiddenNames: [...FORBIDDEN_NAMES, 'propagationCtx'], }; } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/calculated-field-geofencing-zone-groups-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/calculated-field-geofencing-zone-groups-panel.component.ts index 19e54f25dd..adffc5fb7e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/calculated-field-geofencing-zone-groups-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/calculated-field-geofencing-zone-groups-panel.component.ts @@ -32,11 +32,14 @@ import { ArgumentEntityTypeTranslations, CalculatedFieldGeofencing, CalculatedFieldGeofencingValue, + FORBIDDEN_NAMES, + forbiddenNamesValidator, GeofencingDirectionLevelTranslations, GeofencingDirectionTranslations, GeofencingReportStrategy, GeofencingReportStrategyTranslations, - getCalculatedFieldCurrentEntityFilter + getCalculatedFieldCurrentEntityFilter, + uniqueNameValidator } from '@shared/models/calculated-field.models'; import { debounceTime, distinctUntilChanged, map } from 'rxjs/operators'; import { EntityType } from '@shared/models/entity-type.models'; @@ -75,7 +78,7 @@ export class CalculatedFieldGeofencingZoneGroupsPanelComponent implements OnInit readonly maxRelationLevelPerCfArgument = getCurrentAuthState(this.store).maxRelationLevelPerCfArgument; geofencingFormGroup = this.fb.group({ - name: ['', [Validators.required, this.uniqNameRequired(), this.forbiddenNameValidator(), Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], + name: ['', [Validators.required, forbiddenNamesValidator(FORBIDDEN_NAMES), Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], refEntityId: this.fb.group({ entityType: [ArgumentEntityType.Current], id: [''] @@ -132,6 +135,7 @@ export class CalculatedFieldGeofencingZoneGroupsPanelComponent implements OnInit } ngOnInit(): void { + this.updatedFormValidators(); this.geofencingFormGroup.patchValue(this.zone, {emitEvent: false}); if (this.zone.refDynamicSourceConfiguration?.type) { this.refEntityIdFormGroup.get('entityType').setValue(this.zone.refDynamicSourceConfiguration.type, {emitEvent: false}); @@ -158,6 +162,11 @@ export class CalculatedFieldGeofencingZoneGroupsPanelComponent implements OnInit return of(['Contains', 'Manages']).pipe(map(name => name?.filter(option => option.toLowerCase().includes(search)))); } + private updatedFormValidators(): void { + this.geofencingFormGroup.get('name').addValidators(uniqueNameValidator(this.usedNames)); + this.geofencingFormGroup.get('name').updateValueAndValidity({emitEvent: false}); + } + private observeCreateRelationZonesChanges(): void { this.geofencingFormGroup.get('createRelationsWithMatchedZones').valueChanges .pipe(takeUntilDestroyed()) @@ -264,29 +273,12 @@ export class CalculatedFieldGeofencingZoneGroupsPanelComponent implements OnInit }); } - private uniqNameRequired(): ValidatorFn { - return (control: FormControl) => { - const newName = control.value.trim().toLowerCase(); - const isDuplicate = this.usedNames?.some(name => name.toLowerCase() === newName); - - return isDuplicate ? { duplicateName: true } : null; - }; - } - private levelsRequired(): ValidatorFn { return (control: FormControl) => { return control.value.length ? null : { levelsRequired: true }; }; } - private forbiddenNameValidator(): ValidatorFn { - return (control: FormControl) => { - const trimmedValue = control.value.trim().toLowerCase(); - const forbiddenNames = ['ctx', 'e', 'pi']; - return forbiddenNames.includes(trimmedValue) ? { forbiddenName: true } : null; - }; - } - levelsFormArray(): UntypedFormArray { return this.refDynamicSourceFormGroup.get('levels') as UntypedFormArray; } diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index c792c2d06f..432f826ebc 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -30,7 +30,9 @@ import { endGroupHighlightRule } from '@shared/models/ace/ace.models'; import { EntitySearchDirection } from '@shared/models/relation.models'; -import { AbstractControl, ValidationErrors, ValidatorFn } from '@angular/forms'; +import { AbstractControl, FormControl, ValidationErrors, ValidatorFn } from '@angular/forms'; + +export const FORBIDDEN_NAMES = ['ctx', 'e', 'pi']; interface BaseCalculatedField extends Omit, 'label'>, HasVersion, HasEntityDebugSettings, HasTenantId, ExportableEntity { entityId: EntityId; @@ -904,3 +906,26 @@ export function notEmptyObjectValidator(): ValidatorFn { return null; }; } + +export function forbiddenNamesValidator(forbiddenNames: string[]): ValidatorFn { + const forbiddenNameSet = new Set(forbiddenNames); + + return (control: FormControl) => { + const trimmedValue = (control.value || '').trim(); + return forbiddenNameSet.has(trimmedValue) ? { forbiddenName: true } : null; + }; +} + +export function uniqueNameValidator(existingNames: string[]): ValidatorFn { + const namesSet = new Set((existingNames || []).map(name => name.toLowerCase())); + + return (control: FormControl) => { + const newName = (control.value || '').trim().toLowerCase(); + + if (!newName) { + return null; + } + + return namesSet.has(newName) ? { duplicateName: true } : null; + }; +} From fe45f40209b1d940955aa08277e645c587769298 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Mon, 3 Nov 2025 13:15:24 +0200 Subject: [PATCH 15/40] UI: Calculated field Entity aggregation change model and default value --- ...ntity-aggregation-component.component.html | 26 +++++++----- .../entity-aggregation-component.component.ts | 34 ++++++++-------- ...culated-field-metrics-panel.component.html | 8 ++++ ...alculated-field-metrics-panel.component.ts | 40 +++++++++---------- .../calculated-field-metrics-table.module.ts | 1 - .../shared/models/calculated-field.models.ts | 5 ++- .../assets/locale/locale.constant-en_US.json | 14 ++++--- 7 files changed, 71 insertions(+), 57 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.html index 84dcb23d4c..5114a4cdc1 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.html @@ -62,30 +62,32 @@
@if (entityAggregationConfiguration.get('interval.type').value === AggIntervalType.CUSTOM) { + formControlName="durationSec"> }
- +
{{ 'calculated-fields.entity-aggregation.apply-offset' | translate }}
- @if (entityAggregationConfiguration.get('interval.allowOffsetMillis').value) { + @if (entityAggregationConfiguration.get('interval.allowOffsetSec').value) { + formControlName="offsetSec"> }
@@ -99,21 +101,23 @@ @if (entityAggregationConfiguration.get('allowWatermark').value) { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts index 9f48187b55..5fd958518b 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts @@ -41,7 +41,7 @@ import { HOUR, MINUTE, SECOND } from '@shared/models/time/time.models'; import { isDefinedAndNotNull } from '@core/utils'; interface CalculatedFieldEntityAggregationConfigurationValue extends CalculatedFieldEntityAggregationConfiguration { - interval: AggInterval & {allowOffsetMillis?: boolean}; + interval: AggInterval & {allowOffsetSec?: boolean}; allowWatermark: boolean; } @@ -79,14 +79,14 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor interval: this.fb.group({ type: [AggIntervalType.HOUR], tz: ['', Validators.required], - multiplier: [HOUR/SECOND, Validators.required], - allowOffsetMillis: [false], - offsetMillis: [MINUTE/SECOND, Validators.required], + durationSec: [HOUR/SECOND, Validators.required], + allowOffsetSec: [false], + offsetSec: [MINUTE/SECOND, Validators.required], }), allowWatermark: [false], watermark: this.fb.group({ - duration: [6 * MINUTE / SECOND, Validators.required], - checkInterval: [MINUTE / SECOND, Validators.required], + duration: [HOUR/SECOND, Validators.required], + checkInterval: [10 * MINUTE / SECOND, Validators.required], }), output: this.fb.control({ type: OutputType.Timeseries, @@ -111,7 +111,7 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor this.checkAggIntervalType(type); }); - this.entityAggregationConfiguration.get('interval.allowOffsetMillis').valueChanges.pipe( + this.entityAggregationConfiguration.get('interval.allowOffsetSec').valueChanges.pipe( takeUntilDestroyed() ).subscribe((allow: boolean) => { this.checkIntervalDuration(allow); @@ -138,11 +138,11 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor const data: CalculatedFieldEntityAggregationConfigurationValue = { ...value, allowWatermark: isDefinedAndNotNull(value.watermark), - interval: {...value.interval, allowOffsetMillis: isDefinedAndNotNull(value?.interval?.offsetMillis)} + interval: {...value.interval, allowOffsetSec: isDefinedAndNotNull(value?.interval?.offsetSec)} } this.entityAggregationConfiguration.patchValue(data, {emitEvent: false}); this.checkAggIntervalType(this.entityAggregationConfiguration.get('interval.type').value); - this.checkIntervalDuration(this.entityAggregationConfiguration.get('interval.allowOffsetMillis').value); + this.checkIntervalDuration(this.entityAggregationConfiguration.get('interval.allowOffsetSec').value); this.checkWatermark(this.entityAggregationConfiguration.get('allowWatermark').value); setTimeout(() => { this.entityAggregationConfiguration.get('arguments').updateValueAndValidity({onlySelf: true}); @@ -161,17 +161,17 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor } else { this.entityAggregationConfiguration.enable({emitEvent: false}); this.checkAggIntervalType(this.entityAggregationConfiguration.get('interval.type').value); - this.checkIntervalDuration(this.entityAggregationConfiguration.get('interval.allowOffsetMillis').value); + this.checkIntervalDuration(this.entityAggregationConfiguration.get('interval.allowOffsetSec').value); this.checkWatermark(this.entityAggregationConfiguration.get('allowWatermark').value); } } private updatedModel(value: CalculatedFieldEntityAggregationConfigurationValue): void { value.type = CalculatedFieldType.ENTITY_AGGREGATION; - if (!value.interval.allowOffsetMillis) { - delete value.interval.offsetMillis; + if (!value.interval.allowOffsetSec) { + delete value.interval.offsetSec; } - delete value.interval.offsetMillis; + delete value.interval.offsetSec; if (!value.allowWatermark) { delete value.watermark; } @@ -181,17 +181,17 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor private checkAggIntervalType(type: AggIntervalType) { if (type === AggIntervalType.CUSTOM) { - this.entityAggregationConfiguration.get('interval.multiplier').enable({emitEvent: false}); + this.entityAggregationConfiguration.get('interval.durationSec').enable({emitEvent: false}); } else { - this.entityAggregationConfiguration.get('interval.multiplier').disable({emitEvent: false}); + this.entityAggregationConfiguration.get('interval.durationSec').disable({emitEvent: false}); } } private checkIntervalDuration(allow: boolean) { if (allow) { - this.entityAggregationConfiguration.get('interval.offsetMillis').enable({emitEvent: false}); + this.entityAggregationConfiguration.get('interval.offsetSec').enable({emitEvent: false}); } else { - this.entityAggregationConfiguration.get('interval.offsetMillis').disable({emitEvent: false}); + this.entityAggregationConfiguration.get('interval.offsetSec').disable({emitEvent: false}); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-panel.component.html index d57a487dbd..eda8a779ad 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/metrics/calculated-field-metrics-panel.component.html @@ -160,6 +160,14 @@ } + @if (simpleMode) { +
+
{{ 'calculated-fields.default-value' | translate }}
+ + + +
+ }
@if (entityAggregationConfiguration.get('interval.type').value === AggIntervalType.CUSTOM) { } @@ -84,6 +87,7 @@ sameWidthInputs appearance="outline" subscriptSizing="dynamic" + containerClass="flex gap-3" labelText="{{ 'calculated-fields.entity-aggregation.offset-value' | translate }}" minErrorText="{{ 'calculated-fields.entity-aggregation.offset-value-min' | translate }}" requiredText="{{ 'calculated-fields.entity-aggregation.offset-value-required' | translate }}" @@ -104,24 +108,13 @@ [minTime]="60" sameWidthInputs appearance="outline" + containerClass="flex gap-3" labelText="{{ 'calculated-fields.entity-aggregation.duration' | translate }}" minErrorText="{{ 'calculated-fields.entity-aggregation.duration-min' | translate }}" hintText="{{ 'calculated-fields.entity-aggregation.duration-hint' | translate }}" requiredText="{{ 'calculated-fields.entity-aggregation.duration-required' | translate }}" formControlName="duration"> - - }
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts index 5fd958518b..44aba5b6fa 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts @@ -37,8 +37,11 @@ import { } from '@shared/models/calculated-field.models'; import { map } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; -import { HOUR, MINUTE, SECOND } from '@shared/models/time/time.models'; +import { DAY, HOUR, MINUTE, SECOND } from '@shared/models/time/time.models'; import { isDefinedAndNotNull } from '@core/utils'; +import { getCurrentAuthState } from '@core/auth/auth.selectors'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; interface CalculatedFieldEntityAggregationConfigurationValue extends CalculatedFieldEntityAggregationConfiguration { interval: AggInterval & {allowOffsetSec?: boolean}; @@ -72,6 +75,8 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor @Input({required: true}) entityName: string; + readonly minAggregationIntervalInSecForCF = getCurrentAuthState(this.store).minAggregationIntervalInSecForCF; + readonly DayInSec = DAY / SECOND; entityAggregationConfiguration = this.fb.group({ arguments: this.fb.control({}, notEmptyObjectValidator()), @@ -79,14 +84,13 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor interval: this.fb.group({ type: [AggIntervalType.HOUR], tz: ['', Validators.required], - durationSec: [HOUR/SECOND, Validators.required], + durationSec: [this.minAggregationIntervalInSecForCF, Validators.required], allowOffsetSec: [false], offsetSec: [MINUTE/SECOND, Validators.required], }), allowWatermark: [false], watermark: this.fb.group({ duration: [HOUR/SECOND, Validators.required], - checkInterval: [10 * MINUTE / SECOND, Validators.required], }), output: this.fb.control({ type: OutputType.Timeseries, @@ -103,7 +107,8 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor private propagateChange: (config: CalculatedFieldEntityAggregationConfiguration) => void = () => { }; - constructor(private fb: FormBuilder) { + constructor(private fb: FormBuilder, + private store: Store) { this.entityAggregationConfiguration.get('interval.type').valueChanges.pipe( takeUntilDestroyed() diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html index 8cfa8fd0ed..2c2f7b5973 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -342,17 +342,17 @@
- tenant-profile.relation-search-entity-limit - tenant-profile.min-allowed-aggregation-interval + - - {{ 'tenant-profile.relation-search-entity-limit-required' | translate}} + + {{ 'tenant-profile.min-allowed-aggregation-interval-required' | translate}} - - {{ 'tenant-profile.relation-search-entity-limit-range' | translate}} + + {{ 'tenant-profile.min-allowed-aggregation-interval-range' | translate}} - tenant-profile.relation-search-entity-limit-hint + tenant-profile.min-allowed-deduplication-interval @@ -368,6 +368,22 @@
+
+ + tenant-profile.relation-search-entity-limit + + + {{ 'tenant-profile.relation-search-entity-limit-required' | translate}} + + + {{ 'tenant-profile.relation-search-entity-limit-range' | translate}} + + tenant-profile.relation-search-entity-limit-hint + +
+
diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts index 0000d01995..16997d18a7 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts @@ -117,6 +117,7 @@ export class DefaultTenantProfileConfigurationComponent implements ControlValueA maxArgumentsPerCF: [0, [Validators.required, Validators.min(0)]], maxRelationLevelPerCfArgument: [1, [Validators.required, Validators.min(1)]], minAllowedDeduplicationIntervalInSecForCF: [0, [Validators.required, Validators.min(0)]], + minAggregationIntervalInSecForCF: [0, [Validators.required, Validators.min(0)]], maxRelatedEntitiesToReturnPerCfArgument: [1, [Validators.required, Validators.min(1)]], minAllowedScheduledUpdateIntervalInSecForCF: [0, [Validators.required, Validators.min(0)]], maxDataPointsPerRollingArg: [0, [Validators.required, Validators.min(0)]], diff --git a/ui-ngx/src/app/shared/components/time-unit-input.component.html b/ui-ngx/src/app/shared/components/time-unit-input.component.html index f81fc68777..c49ce517b3 100644 --- a/ui-ngx/src/app/shared/components/time-unit-input.component.html +++ b/ui-ngx/src/app/shared/components/time-unit-input.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+
diff --git a/ui-ngx/src/app/shared/components/time-unit-input.component.ts b/ui-ngx/src/app/shared/components/time-unit-input.component.ts index bcfdabd574..65acaa88f1 100644 --- a/ui-ngx/src/app/shared/components/time-unit-input.component.ts +++ b/ui-ngx/src/app/shared/components/time-unit-input.component.ts @@ -23,6 +23,7 @@ import { NG_VALUE_ACCESSOR, ValidationErrors, Validator, + ValidatorFn, Validators } from '@angular/forms'; import { TimeUnit, timeUnitTranslations } from '@home/components/rule-node/rule-node-config.models'; @@ -79,6 +80,13 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, @Input() maxErrorText: string; + @Input() + @coerceNumber() + stepMultipleOf: number; + + @Input() + stepMultipleOfErrorText: string; + @Input() subscriptSizing: SubscriptSizing = 'fixed'; @@ -93,6 +101,9 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, @coerceBoolean() sameWidthInputs: boolean = false; + @Input() + containerClass: string | string[] | Record = "flex gap-4"; + timeUnits = Object.values(TimeUnit).filter(item => item !== TimeUnit.MILLISECONDS) as TimeUnit[]; timeUnitTranslations = timeUnitTranslations; @@ -128,7 +139,7 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, this.timeUnits = this.timeUnits.filter(item => item !== TimeUnit.DAYS); } } - if(this.required || this.maxTime) { + if (this.required || this.maxTime || isDefinedAndNotNull(this.minTime) || this.stepMultipleOf) { const timeControl = this.timeInputForm.get('time'); const validators = [Validators.pattern(/^\d*$/)]; if (this.required) { @@ -145,6 +156,10 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, ); } + if (isDefinedAndNotNull(this.stepMultipleOf) && this.stepMultipleOf > 0) { + validators.push(this.createStepMultipleOfValidator()); + } + timeControl.setValidators(validators); timeControl.updateValueAndValidity({ emitEvent: false }); } @@ -170,6 +185,8 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, return this.minErrorText; } else if (this.timeInputForm.get('time').hasError('max') && this.maxErrorText) { return this.maxErrorText; + } else if (this.timeInputForm.get('time').hasError('stepMultipleOf') && this.stepMultipleOfErrorText) { + return this.stepMultipleOfErrorText; } } @@ -232,4 +249,34 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, } } + private createStepMultipleOfValidator(): ValidatorFn { + return (control: AbstractControl): ValidationErrors | null => { + const time = control.value; + if (!isDefinedAndNotNull(time) || !isNumeric(time)) { + return null; + } + const numericTime = Number(time); + if (numericTime === 0) { + return null; + } + + const timeUnit = control.parent?.get('timeUnit')?.value as TimeUnit; + if (!timeUnit) { + return null; + } + + const unitInSec = this.timeIntervalsInSec.get(timeUnit); + const totalTimeInSec = numericTime * unitInSec; + const multipleOfVal = this.stepMultipleOf; + + let isValid: boolean; + if (totalTimeInSec < multipleOfVal) { + isValid = (multipleOfVal % totalTimeInSec === 0); + } else { + isValid = (totalTimeInSec % multipleOfVal === 0); + } + return isValid ? null : { stepMultipleOf: true }; + }; + } + } diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 321c176cde..e0df001e3f 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -135,8 +135,7 @@ export interface CalculatedFieldEntityAggregationConfiguration { } export interface WatermarkConfig { - duration?: number; - checkInterval?: number; + duration: number; } interface BasePropagationConfiguration { diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index ae7a0ae8b8..ac7b77b9cc 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -108,6 +108,7 @@ export interface DefaultTenantProfileConfiguration { maxArgumentsPerCF: number; maxRelationLevelPerCfArgument: number; minAllowedDeduplicationIntervalInSecForCF: number; + minAggregationIntervalInSecForCF: number; maxRelatedEntitiesToReturnPerCfArgument: number; minAllowedScheduledUpdateIntervalInSecForCF: number; maxDataPointsPerRollingArg: number; @@ -176,6 +177,7 @@ export function createTenantProfileConfiguration(type: TenantProfileType): Tenan maxDataPointsPerRollingArg: 1000, maxRelationLevelPerCfArgument: 10, minAllowedDeduplicationIntervalInSecForCF: 3600, + minAggregationIntervalInSecForCF: 60, maxRelatedEntitiesToReturnPerCfArgument: 100, minAllowedScheduledUpdateIntervalInSecForCF: 0, maxStateSizeInKBytes: 32, diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 6e00b883b4..c05686b648 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1195,7 +1195,8 @@ "aggregate-interval-type": "Aggregate interval type", "aggregate-interval-value": "Aggregate interval value", "aggregate-interval-value-required": "Aggregate interval value is required", - "aggregate-interval-value-min": "Aggregate interval value should be at least 1 minute", + "aggregate-interval-value-min": "Aggregate interval value should be at least { sec, plural, =0 {0 second} =1 {1 second} other {# seconds} }", + "aggregate-interval-value-step-multiple-of": "Aggregate interval value must be a divisor or multiple of 1 day", "aggregate-period": { "hour": "Hour", "day": "Day", @@ -1220,12 +1221,7 @@ "duration": "Duration", "duration-required": "Duration is required", "duration-min": "Duration should be at least 1 minute", - "duration-hint": "Defines how long to wait for delayed data after the interval ends", - "check-interval": "Check for telemetry every", - "check-interval-required": "Check for telemetry every is required", - "check-interval-min": "Check interval should be at least 30 seconds", - "check-interval-max": "Check interval need be less than the duration", - "check-interval-hint": "Defines how often to recheck for late telemetry during the watermark period" + "duration-hint": "How long to wait for delayed data after the interval ends" }, "hint": { "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with time series rolling type.", @@ -6001,6 +5997,9 @@ "min-allowed-deduplication-interval": "Min allowed deduplication interval (seconds)", "min-allowed-deduplication-interval-range": "Min allowed deduplication interval value can't be negative", "min-allowed-deduplication-interval-required": "Min allowed deduplication interval is required", + "min-allowed-aggregation-interval": "Min allowed aggregation interval (seconds)", + "min-allowed-aggregation-interval-range": "Min allowed aggregation interval value can't be negative", + "min-allowed-aggregation-interval-required": "Min allowed aggregation interval is required", "min-allowed-scheduled-update-interval-required": "Min allowed update interval min number is required", "max-state-size": "State maximum size in KB", "max-state-size-range": "State maximum size in KB can't be negative", From 4037efaf3a1137a342ec05d043ccbcebace9bc89 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 6 Nov 2025 09:18:28 +0200 Subject: [PATCH 21/40] round result --- .../EntityAggregationCalculatedFieldState.java | 12 ++++++++---- .../validator/CalculatedFieldDataValidator.java | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java index 5e45b97f9a..af7912da46 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; @@ -89,11 +90,11 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt }); removeExpiredIntervals(expiredIntervals); - ArrayNode result = toResult(results); + Output output = ctx.getOutput(); + ArrayNode result = toResult(results, output.getDecimalsByDefault()); if (result.isEmpty()) { return Futures.immediateFuture(TelemetryCalculatedFieldResult.EMPTY); } - Output output = ctx.getOutput(); return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() .type(output.getType()) .scope(output.getScope()) @@ -227,7 +228,7 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt .orElse(null); } - protected ArrayNode toResult(Map> results) { + protected ArrayNode toResult(Map> results, Integer precision) { ArrayNode result = JacksonUtil.newArrayNode(); results.forEach((interval, args) -> { ObjectNode metricsNode = JacksonUtil.newObjectNode(); @@ -235,7 +236,10 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt String metricName = entry.getKey(); ArgumentEntry argumentEntry = entry.getValue(); if (!argumentEntry.isEmpty()) { - metricsNode.put(metricName, JacksonUtil.toString(argumentEntry.getValue())); + Object resultValue = argumentEntry.getValue() instanceof Number number + ? TbUtils.roundResult(number.doubleValue(), precision) + : argumentEntry.getValue(); + metricsNode.put(metricName, JacksonUtil.toString(resultValue)); } } ObjectNode resultNode = JacksonUtil.newObjectNode(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index 7813a244e1..087640eda5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -134,7 +134,7 @@ public class CalculatedFieldDataValidator extends DataValidator if (minAggregationIntervalInSec <= 0) { return; } - if (aggConfiguration.getInterval().getIntervalDurationMillis() > TimeUnit.SECONDS.toMillis(minAggregationIntervalInSec)) { + if (aggConfiguration.getInterval().getIntervalDurationMillis() < TimeUnit.SECONDS.toMillis(minAggregationIntervalInSec)) { throw new IllegalArgumentException("Aggregation interval duration is less than configured " + "minimum allowed aggregation interval in tenant profile: " + minAggregationIntervalInSec + " sec."); } From 11983660f735d5941f11b00856db32415880146b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 6 Nov 2025 09:41:13 +0200 Subject: [PATCH 22/40] added quarter interval type --- .../single/interval/AggInterval.java | 1 + .../single/interval/AggIntervalType.java | 1 + .../single/interval/BaseAggInterval.java | 12 ++++++++ .../single/interval/MonthInterval.java | 1 - .../single/interval/QuarterInterval.java | 30 +++++++++++++++++++ 5 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/QuarterInterval.java diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java index d6d1aca6a8..b534002885 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java @@ -31,6 +31,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = WeekInterval.class, name = "WEEK"), @JsonSubTypes.Type(value = WeekSunSatInterval.class, name = "WEEK_SUN_SAT"), @JsonSubTypes.Type(value = MonthInterval.class, name = "MONTH"), + @JsonSubTypes.Type(value = QuarterInterval.class, name = "QUARTER"), @JsonSubTypes.Type(value = YearInterval.class, name = "YEAR"), @JsonSubTypes.Type(value = CustomInterval.class, name = "CUSTOM") }) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java index 62185127ed..1d39f14a03 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalType.java @@ -22,6 +22,7 @@ public enum AggIntervalType { WEEK, WEEK_SUN_SAT, MONTH, + QUARTER, YEAR, CUSTOM diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java index 08bea68d5b..71ccab5f95 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java @@ -47,6 +47,7 @@ public abstract class BaseAggInterval implements AggInterval { case DAY -> Duration.ofDays(1).toMillis(); case WEEK, WEEK_SUN_SAT -> Duration.ofDays(7L).toMillis(); case MONTH -> Duration.ofDays(Math.round(30)).toMillis(); // average + case QUARTER -> Duration.ofDays(Math.round(91)).toMillis(); case YEAR -> Duration.ofDays(Math.round(365)).toMillis(); default -> throw new IllegalArgumentException("Unsupported type: " + getType()); }; @@ -88,6 +89,7 @@ public abstract class BaseAggInterval implements AggInterval { case WEEK -> alignByWeeks(reference, DayOfWeek.MONDAY, next); case WEEK_SUN_SAT -> alignByWeeks(reference, DayOfWeek.SUNDAY, next); case MONTH -> alignByMonths(reference, next); + case QUARTER -> alignByQuarters(reference, next); case YEAR -> alignByYears(reference, next); default -> throw new IllegalArgumentException("Unsupported interval type: " + getType()); }; @@ -114,6 +116,16 @@ public abstract class BaseAggInterval implements AggInterval { return next ? base.plusMonths(1) : base; } + private ZonedDateTime alignByQuarters(ZonedDateTime now, boolean next) { + int month = now.getMonthValue(); + int quarterStartMonth = ((month - 1) / 3) * 3 + 1; // 1, 4, 7, 10 + ZonedDateTime base = ZonedDateTime.of( + LocalDate.of(now.getYear(), quarterStartMonth, 1), + LocalTime.MIDNIGHT, + now.getZone()); + return next ? base.plusMonths(3) : base; + } + private ZonedDateTime alignByYears(ZonedDateTime now, boolean next) { ZonedDateTime base = ZonedDateTime.of( LocalDate.of(now.getYear(), 1, 1), diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java index 7225eaca9f..91fc3d0413 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java @@ -27,5 +27,4 @@ public class MonthInterval extends BaseAggInterval { return AggIntervalType.MONTH; } - } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/QuarterInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/QuarterInterval.java new file mode 100644 index 0000000000..eb774b2341 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/QuarterInterval.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +public class QuarterInterval extends BaseAggInterval { + + @Override + public AggIntervalType getType() { + return AggIntervalType.QUARTER; + } + +} From 7d48aca10d8d8285fb70367a6345a4b2add2df38 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 6 Nov 2025 18:58:14 +0200 Subject: [PATCH 23/40] UI: Fixed entity-aggregation-component.component.ts --- .../entity-aggregation-component.component.ts | 2 +- ui-ngx/src/app/shared/models/calculated-field.models.ts | 2 ++ ui-ngx/src/assets/locale/locale.constant-en_US.json | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts index 44aba5b6fa..81fb709f99 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts @@ -176,7 +176,7 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor if (!value.interval.allowOffsetSec) { delete value.interval.offsetSec; } - delete value.interval.offsetSec; + delete value.interval.allowOffsetSec; if (!value.allowWatermark) { delete value.watermark; } diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index e0df001e3f..cfb7d38565 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -294,6 +294,7 @@ export enum AggIntervalType { WEEK = 'WEEK', WEEK_SUN_SAT = 'WEEK_SUN_SAT', MONTH = 'MONTH', + QUARTER = 'QUARTER', YEAR = 'YEAR', CUSTOM = 'CUSTOM' } @@ -305,6 +306,7 @@ export const AggIntervalTypeTranslations = new Map( [AggIntervalType.WEEK, 'calculated-fields.aggregate-period.week'], [AggIntervalType.WEEK_SUN_SAT, 'calculated-fields.aggregate-period.week-sun-sat'], [AggIntervalType.MONTH, 'calculated-fields.aggregate-period.month'], + [AggIntervalType.QUARTER, 'calculated-fields.aggregate-period.quarter'], [AggIntervalType.YEAR, 'calculated-fields.aggregate-period.year'], [AggIntervalType.CUSTOM, 'calculated-fields.aggregate-period.custom'] ] diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index c05686b648..cff76239be 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1203,6 +1203,7 @@ "week": "Week (Mon - Sun)", "week-sun-sat": "Week (Sun - Sat)", "month": "Month", + "quarter": "Quarter", "year": "Year", "custom": "Custom" }, From 2dac911d5f2ad3068bf359792a1e75adcf3a0ffa Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 7 Nov 2025 15:16:58 +0200 Subject: [PATCH 24/40] UI: Rename Entity aggregation to Time series data aggregation --- ui-ngx/src/app/shared/models/calculated-field.models.ts | 3 ++- ui-ngx/src/assets/locale/locale.constant-en_US.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 3875e2b3eb..77ba570204 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -106,7 +106,8 @@ export const CalculatedFieldTypeTranslations = new Map Date: Mon, 10 Nov 2025 19:45:25 +0200 Subject: [PATCH 25/40] UI: Add offse hint in CF; Improvement time unit component --- ...ntity-aggregation-component.component.html | 7 +- .../entity-aggregation-component.component.ts | 209 +++++++++++++++++- .../components/time-unit-input.component.ts | 42 +++- .../assets/locale/locale.constant-en_US.json | 3 + 4 files changed, 245 insertions(+), 16 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.html index 1b45b7f245..2907472143 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.html @@ -71,7 +71,7 @@ labelText="{{ 'calculated-fields.aggregate-interval-value' | translate }}" minErrorText="{{ 'calculated-fields.aggregate-interval-value-min' | translate : {sec: minAggregationIntervalInSecForCF} }}" requiredText="{{ 'calculated-fields.aggregate-interval-value-required' | translate }}" - stepMultipleOfErrorText="Must be 1 day" + stepMultipleOfErrorText="{{ 'calculated-fields.aggregate-interval-value-step-multiple-of' | translate }}" formControlName="durationSec"> } @@ -84,15 +84,20 @@ @if (entityAggregationConfiguration.get('interval.allowOffsetSec').value) { +
+ {{ hint }} +
}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts index 81fb709f99..8da5c47606 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts @@ -35,19 +35,29 @@ import { notEmptyObjectValidator, OutputType } from '@shared/models/calculated-field.models'; -import { map } from 'rxjs/operators'; +import { filter, map } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; -import { DAY, HOUR, MINUTE, SECOND } from '@shared/models/time/time.models'; +import { AVG_MONTH, AVG_QUARTER, DAY, HOUR, MINUTE, SECOND, YEAR } from '@shared/models/time/time.models'; import { isDefinedAndNotNull } from '@core/utils'; import { getCurrentAuthState } from '@core/auth/auth.selectors'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; +import { merge } from 'rxjs'; +import { TranslateService } from '@ngx-translate/core'; +import _moment from 'moment'; interface CalculatedFieldEntityAggregationConfigurationValue extends CalculatedFieldEntityAggregationConfiguration { interval: AggInterval & {allowOffsetSec?: boolean}; allowWatermark: boolean; } +enum TimeCategory { + SECONDS = 'SECONDS', + MINUTES = 'MINUTES', + HOURS = 'HOURS', + DAYS = 'DAYS' +} + @Component({ selector: 'tb-entity-aggregation-component', templateUrl: './entity-aggregation-component.component.html', @@ -86,7 +96,7 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor tz: ['', Validators.required], durationSec: [this.minAggregationIntervalInSecForCF, Validators.required], allowOffsetSec: [false], - offsetSec: [MINUTE/SECOND, Validators.required], + offsetSec: [this.minAggregationIntervalInSecForCF > 60 ? MINUTE / SECOND : 1, Validators.required], }), allowWatermark: [false], watermark: this.fb.group({ @@ -105,10 +115,13 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor AggIntervalTypes = Object.values(AggIntervalType) as AggIntervalType[]; AggIntervalTypeTranslations = AggIntervalTypeTranslations; + hint: string; + private propagateChange: (config: CalculatedFieldEntityAggregationConfiguration) => void = () => { }; constructor(private fb: FormBuilder, - private store: Store) { + private store: Store, + private translate: TranslateService,) { this.entityAggregationConfiguration.get('interval.type').valueChanges.pipe( takeUntilDestroyed() @@ -128,6 +141,18 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor this.checkWatermark(allow); }); + merge( + this.entityAggregationConfiguration.get('interval.type').valueChanges, + this.entityAggregationConfiguration.get('interval.durationSec').valueChanges, + this.entityAggregationConfiguration.get('interval.offsetSec').valueChanges, + this.entityAggregationConfiguration.get('interval.allowOffsetSec').valueChanges, + ).pipe( + filter(() => this.entityAggregationConfiguration.get('interval.allowOffsetSec').value), + takeUntilDestroyed() + ).subscribe(() => { + this.updatedOffsetHint(); + }); + this.entityAggregationConfiguration.valueChanges.pipe( takeUntilDestroyed() ).subscribe((value: CalculatedFieldEntityAggregationConfigurationValue) => { @@ -149,6 +174,7 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor this.checkAggIntervalType(this.entityAggregationConfiguration.get('interval.type').value); this.checkIntervalDuration(this.entityAggregationConfiguration.get('interval.allowOffsetSec').value); this.checkWatermark(this.entityAggregationConfiguration.get('allowWatermark').value); + this.updatedOffsetHint(); setTimeout(() => { this.entityAggregationConfiguration.get('arguments').updateValueAndValidity({onlySelf: true}); }); @@ -171,6 +197,26 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor } } + get maxOffsetTime(): number { + switch (this.entityAggregationConfiguration.get('interval.type').value as AggIntervalType) { + case AggIntervalType.HOUR: + return HOUR / SECOND - 1; + case AggIntervalType.DAY: + return DAY / SECOND - 1; + case AggIntervalType.WEEK: + case AggIntervalType.WEEK_SUN_SAT: + return 7 * DAY / SECOND - 1; + case AggIntervalType.MONTH: + return AVG_MONTH / SECOND; + case AggIntervalType.QUARTER: + return AVG_QUARTER / SECOND - 1; + case AggIntervalType.YEAR: + return YEAR / SECOND - 1; + case AggIntervalType.CUSTOM: + return this.entityAggregationConfiguration.get('interval.durationSec').value - 1; + } + } + private updatedModel(value: CalculatedFieldEntityAggregationConfigurationValue): void { value.type = CalculatedFieldType.ENTITY_AGGREGATION; if (!value.interval.allowOffsetSec) { @@ -197,6 +243,7 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor this.entityAggregationConfiguration.get('interval.offsetSec').enable({emitEvent: false}); } else { this.entityAggregationConfiguration.get('interval.offsetSec').disable({emitEvent: false}); + this.hint = ''; } } @@ -207,4 +254,158 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor this.entityAggregationConfiguration.get('watermark').disable({emitEvent: false}); } } + + private updatedOffsetHint(): void { + const offset = this.entityAggregationConfiguration.get('interval.offsetSec').value; + const intervalType = this.entityAggregationConfiguration.get('interval.type').value as AggIntervalType; + const durationSec = this.entityAggregationConfiguration.get('interval.durationSec').value; + const offsetCategory = this.getTimeCategory(offset); + const now = _moment.utc(); + let interval: string = ''; + if (intervalType === AggIntervalType.CUSTOM) { + const durationSecCategory = this.getTimeCategory(durationSec); + const formatString = this.getCustomFormatString(offsetCategory, durationSecCategory); + const intervals: string[] = []; + let allInterval = durationSec >= HOUR*6/SECOND && durationSec < DAY/SECOND; + now.startOf('year').add(offset, 'seconds'); + + let repeat = 2; + if (allInterval) { + repeat = Math.floor(DAY/SECOND/durationSec); + if (repeat > 4) { + repeat = 2; + allInterval = false; + } + } + + for (let i = 0; i < repeat; i++) { + const s1 = now.clone().add(i * durationSec, 'seconds').format(formatString); + const s2 = now.clone().add((i + 1) * durationSec, 'seconds').format(formatString); + intervals.push(`${s1} - ${s2}`); + } + interval = intervals.join('; '); + + if (allInterval) { + this.hint = this.translate.instant('calculated-fields.aggregate-period-hint-offset', {interval}); + } else { + interval += '…' + this.hint = this.translate.instant('calculated-fields.aggregate-period-hint-offset-and-so-on', {interval}); + } + } else { + interval = this.buildStandardIntervalString(now, intervalType, offset, offsetCategory); + this.hint = this.translate.instant('calculated-fields.aggregate-period-hint-offset-and-so-on', { interval }); + } + } + + private getTimeCategory(seconds: number): TimeCategory { + if (seconds % (DAY / SECOND) === 0) { + return TimeCategory.DAYS; + } + if (seconds % (HOUR / SECOND) === 0) { + return TimeCategory.HOURS; + } + if (seconds % (MINUTE / SECOND) === 0) { + return TimeCategory.MINUTES; + } + return TimeCategory.SECONDS; + } + + private getCustomFormatString(offsetCat: TimeCategory, durationCat: TimeCategory): string { + if (durationCat === TimeCategory.DAYS) { + if (offsetCat === TimeCategory.SECONDS) { + return '[Day] D, HH:mm:ss'; + } + if (offsetCat === TimeCategory.MINUTES || offsetCat === TimeCategory.HOURS) { + return '[Day] D, HH:mm'; + } + return '[Day] D'; + } else { + if (offsetCat === TimeCategory.SECONDS) { + return 'HH:mm:ss'; + } + return 'HH:mm'; + } + } + + private formatAdditiveInterval(now: _moment.Moment, addUnit: 'hour' | 'day' | 'month' | 'quarter', offsetCat: TimeCategory, + formats: { [key in TimeCategory]?: { s1: string, s2: string, s3: string } }): string { + const formatTs = formats[offsetCat] || formats[TimeCategory.SECONDS]; + + if (!formatTs) { + return ''; + } + + const s1 = now.format(formatTs.s1); + const s2 = now.clone().add(1, addUnit).format(formatTs.s2); + const s3 = now.clone().add(2, addUnit).format(formatTs.s3); + + return `${s1} - ${s2}; ${s2} - ${s3}…`; + } + + private formatNextInterval(now: _moment.Moment, offsetCat: TimeCategory, secFmt: string, minHourFmt: string, dayFmt: string): string { + let s1: string; + if (offsetCat === TimeCategory.SECONDS) { + s1 = now.format(secFmt); + } else if (offsetCat === TimeCategory.MINUTES || offsetCat === TimeCategory.HOURS) { + s1 = now.format(minHourFmt); + } else { + s1 = now.format(dayFmt); + } + + const s2 = `Next ${s1}`; + const s3 = `Following ${s1}`; + return `${s1} - ${s2}; ${s2} - ${s3}… `; + } + + private buildStandardIntervalString(now: _moment.Moment, type: AggIntervalType, offset: number, offsetCat: TimeCategory): string { + switch (type) { + case AggIntervalType.HOUR: + now.startOf('day').add(offset, 'seconds'); + return this.formatAdditiveInterval(now, 'hour', offsetCat, { + [TimeCategory.SECONDS]: { s1: 'HH:mm:ss', s2: 'HH:mm:ss', s3: 'HH:mm:ss' }, + [TimeCategory.MINUTES]: { s1: 'HH:mm:ss', s2: 'HH:mm', s3: 'HH:mm' } + }); + + case AggIntervalType.DAY: + now.startOf('month').add(offset, 'seconds'); + return this.formatAdditiveInterval(now, 'day', offsetCat, { + [TimeCategory.SECONDS]: { s1: '[Day] D, HH:mm:ss', s2: '[Day] D, HH:mm:ss', s3: '[Day] D, HH:mm:ss' }, + [TimeCategory.MINUTES]: { s1: '[Day] D, HH:mm:ss', s2: '[Day] D, HH:mm', s3: '[Day] D, HH:mm' }, + [TimeCategory.HOURS]: { s1: 'HH:mm:ss', s2: '[Day] D, HH:mm', s3: '[Day] D, HH:mm' } // Note: Original logic, s1 format is different + }); + + case AggIntervalType.WEEK: + now.isoWeekday(1).startOf('isoWeek').add(offset, 'seconds'); + return this.formatNextInterval(now, offsetCat, 'ddd, HH:mm:ss', 'ddd, HH:mm', 'ddd'); + + case AggIntervalType.WEEK_SUN_SAT: + now.startOf('week').add(offset, 'seconds'); + return this.formatNextInterval(now, offsetCat, 'ddd, HH:mm:ss', 'ddd, HH:mm', 'ddd'); + + case AggIntervalType.MONTH: + now.startOf('year').add(offset, 'seconds'); + return this.formatAdditiveInterval(now, 'month', offsetCat, { + [TimeCategory.SECONDS]: { s1: 'Do [of month], HH:mm:ss', s2: '[Next] Do, HH:mm:ss', s3: '[Following] Do, HH:mm:ss' }, + [TimeCategory.MINUTES]: { s1: 'Do [of month], HH:mm', s2: '[Next] Do, HH:mm', s3: '[Following] Do, HH:mm' }, + [TimeCategory.HOURS]: { s1: 'Do [of month], HH:mm', s2: '[Next] Do, HH:mm', s3: '[Following] Do, HH:mm' }, + [TimeCategory.DAYS]: { s1: 'Do [of month]', s2: '[Next] Do', s3: '[Following] Do' } + }); + + case AggIntervalType.QUARTER: + now.startOf('year').add(offset, 'seconds'); + return this.formatAdditiveInterval(now, 'quarter', offsetCat, { + [TimeCategory.SECONDS]: { s1: 'MMM Do, HH:mm:ss', s2: 'MMM Do, HH:mm:ss', s3: 'MMM Do, HH:mm:ss' }, + [TimeCategory.MINUTES]: { s1: 'MMM Do, HH:mm', s2: 'MMM Do, HH:mm', s3: 'MMM Do, HH:mm' }, + [TimeCategory.HOURS]: { s1: 'MMM Do, HH:mm', s2: 'MMM Do, HH:mm', s3: 'MMM Do, HH:mm' }, + [TimeCategory.DAYS]: { s1: 'MMM Do', s2: 'MMM Do', s3: 'MMM Do' } + }); + + case AggIntervalType.YEAR: + now.startOf('year').add(offset, 'seconds'); + return this.formatNextInterval(now, offsetCat, 'MMM Do, HH:mm:ss', 'MMM Do, HH:mm', 'MMM Do'); + + default: + return ''; + } + } } diff --git a/ui-ngx/src/app/shared/components/time-unit-input.component.ts b/ui-ngx/src/app/shared/components/time-unit-input.component.ts index 65acaa88f1..ac2211a50d 100644 --- a/ui-ngx/src/app/shared/components/time-unit-input.component.ts +++ b/ui-ngx/src/app/shared/components/time-unit-input.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, DestroyRef, forwardRef, Input, OnInit } from '@angular/core'; +import { Component, DestroyRef, forwardRef, Input, OnChanges, OnInit, SimpleChanges } from '@angular/core'; import { AbstractControl, ControlValueAccessor, @@ -51,7 +51,7 @@ interface TimeUnitInputModel { multi: true }] }) -export class TimeUnitInputComponent implements ControlValueAccessor, Validator, OnInit { +export class TimeUnitInputComponent implements ControlValueAccessor, Validator, OnInit, OnChanges { @Input() labelText: string; @@ -129,15 +129,8 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, } ngOnInit() { - if (this.maxTime) { - const maxTimeMs = this.maxTime * SECOND; - if (maxTimeMs < MINUTE) { - this.timeUnits = this.timeUnits.filter(item => item !== TimeUnit.MINUTES && item !== TimeUnit.HOURS && item !== TimeUnit.DAYS); - } else if (maxTimeMs < HOUR) { - this.timeUnits = this.timeUnits.filter(item => item !== TimeUnit.HOURS && item !== TimeUnit.DAYS); - } else if (maxTimeMs < DAY) { - this.timeUnits = this.timeUnits.filter(item => item !== TimeUnit.DAYS); - } + if (isDefinedAndNotNull(this.maxTime)) { + this.updatedAllowTimeUnitInterval(this.maxTime); } if (this.required || this.maxTime || isDefinedAndNotNull(this.minTime) || this.stepMultipleOf) { const timeControl = this.timeInputForm.get('time'); @@ -190,6 +183,21 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, } } + ngOnChanges(changes: SimpleChanges): void { + for (const propName of Object.keys(changes)) { + const change = changes[propName]; + if (!change.firstChange && change.currentValue !== change.previousValue) { + if (propName === 'maxTime') { + if (isDefinedAndNotNull(this.maxTime)) { + this.timeUnits = Object.values(TimeUnit).filter(item => item !== TimeUnit.MILLISECONDS) as TimeUnit[]; + this.updatedAllowTimeUnitInterval(this.maxTime); + this.timeInputForm.get('time').updateValueAndValidity({emitEvent: false}); + } + } + } + } + } + registerOnChange(fn: any) { this.propagateChange = fn; } @@ -279,4 +287,16 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, }; } + private updatedAllowTimeUnitInterval(maxTime: number) { + const maxTimeMs = maxTime * SECOND; + this.timeUnits = Object.values(TimeUnit).filter(item => item !== TimeUnit.MILLISECONDS) as TimeUnit[]; + if (maxTimeMs < MINUTE) { + this.timeUnits = this.timeUnits.filter(item => item !== TimeUnit.MINUTES && item !== TimeUnit.HOURS && item !== TimeUnit.DAYS); + } else if (maxTimeMs < HOUR) { + this.timeUnits = this.timeUnits.filter(item => item !== TimeUnit.HOURS && item !== TimeUnit.DAYS); + } else if (maxTimeMs < DAY) { + this.timeUnits = this.timeUnits.filter(item => item !== TimeUnit.DAYS); + } + } + } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 9d3c2b74eb..1c1f8e7f76 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1210,6 +1210,8 @@ "year": "Year", "custom": "Custom" }, + "aggregate-period-hint-offset": "Your aggregation interval will be: {{ interval }}", + "aggregate-period-hint-offset-and-so-on": "Your aggregation interval will be: {{ interval }} and so on", "entity-aggregation": { "argument-hint": "Data will be fetched from selected entity", "argument-setting-hint": "Latest telemetry is the only available argument type for this calculated field", @@ -1220,6 +1222,7 @@ "offset-value": "Offset value", "offset-value-required": "Offset value is required", "offset-value-min": "Offset value must be a positive integer", + "offset-value-max": "Offset value should be less than the aggregate interval value", "wait-delay": "Wait for delayed telemetry", "wait-delay-hint": "Waits for delayed telemetry after the interval ends", "duration": "Duration", From e6790f23681a48d8beee5281ee8798aa86a01db0 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Mon, 10 Nov 2025 23:05:25 +0200 Subject: [PATCH 26/40] lwm2m: test CID client --- ...ityLwM2MIntegrationDtlsCidLength0Test.java | 6 +- ...tyLwM2MIntegrationDtlsCidLength16Test.java | 39 ++++++++++++ ...tyLwM2MIntegrationDtlsCidLength1Test.java} | 11 ++-- ...ityLwM2MIntegrationDtlsCidLength4Test.java | 39 ++++++++++++ ...ityLwM2MIntegrationDtlsCidLength8Test.java | 39 ++++++++++++ ...LwM2MIntegrationDtlsCidLengthNullTest.java | 5 +- ...rityLwM2MIntegrationDtlsCidLengthTest.java | 39 +++++++++++- ...oSecLwM2MIntegrationDtlsCidLengthTest.java | 19 +++++- .../PskLwm2mIntegrationDtlsCidLengthTest.java | 17 ++++- ...oSecLwM2MIntegrationDtlsCidLengthTest.java | 63 +++++++++++++++++++ .../PskLwm2mIntegrationDtlsCidLengthTest.java | 63 +++++++++++++++++++ ...oSecLwM2MIntegrationDtlsCidLengthTest.java | 63 +++++++++++++++++++ .../PskLwm2mIntegrationDtlsCidLengthTest.java | 63 +++++++++++++++++++ ...oSecLwM2MIntegrationDtlsCidLengthTest.java | 27 ++++++-- .../PskLwm2mIntegrationDtlsCidLengthTest.java | 26 ++++++-- ...oSecLwM2MIntegrationDtlsCidLengthTest.java | 63 +++++++++++++++++++ .../PskLwm2mIntegrationDtlsCidLengthTest.java | 63 +++++++++++++++++++ ...oSecLwM2MIntegrationDtlsCidLengthTest.java | 19 +++++- .../PskLwm2mIntegrationDtlsCidLengthTest.java | 18 +++++- 19 files changed, 652 insertions(+), 30 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength16Test.java rename application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/{AbstractSecurityLwM2MIntegrationDtlsCidLength3Test.java => AbstractSecurityLwM2MIntegrationDtlsCidLength1Test.java} (78%) create mode 100644 application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength4Test.java create mode 100644 application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength8Test.java create mode 100644 application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_1/NoSecLwM2MIntegrationDtlsCidLengthTest.java create mode 100644 application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_1/PskLwm2mIntegrationDtlsCidLengthTest.java create mode 100644 application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_16/NoSecLwM2MIntegrationDtlsCidLengthTest.java create mode 100644 application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_16/PskLwm2mIntegrationDtlsCidLengthTest.java rename application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/{serverDtlsCidLength_3 => serverDtlsCidLength_4}/NoSecLwM2MIntegrationDtlsCidLengthTest.java (73%) rename application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/{serverDtlsCidLength_3 => serverDtlsCidLength_4}/PskLwm2mIntegrationDtlsCidLengthTest.java (73%) create mode 100644 application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_8/NoSecLwM2MIntegrationDtlsCidLengthTest.java create mode 100644 application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_8/PskLwm2mIntegrationDtlsCidLengthTest.java diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength0Test.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength0Test.java index 7a0b0f8580..a3229a7c19 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength0Test.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength0Test.java @@ -29,10 +29,12 @@ import org.thingsboard.server.dao.service.DaoSqlTest; public abstract class AbstractSecurityLwM2MIntegrationDtlsCidLength0Test extends AbstractSecurityLwM2MIntegrationDtlsCidLengthTest { + private static final Integer serverDtlsCidLength = 0; + protected void testNoSecDtlsCidLength(Integer dtlsCidLength) throws Exception { - testNoSecDtlsCidLength(dtlsCidLength, 0); + testNoSecDtlsCidLength(dtlsCidLength, serverDtlsCidLength); } protected void testPskDtlsCidLength(Integer dtlsCidLength) throws Exception { - testPskDtlsCidLength(dtlsCidLength, 0); + testPskDtlsCidLength(dtlsCidLength, serverDtlsCidLength); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength16Test.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength16Test.java new file mode 100644 index 0000000000..1481153d8d --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength16Test.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.security.cid; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.dao.service.DaoSqlTest; + + +@TestPropertySource(properties = { + "transport.lwm2m.dtls.connection_id_length=16" +}) + +@DaoSqlTest +@Slf4j +public abstract class AbstractSecurityLwM2MIntegrationDtlsCidLength16Test extends AbstractSecurityLwM2MIntegrationDtlsCidLengthTest { + + private static final Integer serverDtlsCidLength = 16; + + protected void testNoSecDtlsCidLength(Integer dtlsCidLength) throws Exception { + testNoSecDtlsCidLength(dtlsCidLength, serverDtlsCidLength); + } + protected void testPskDtlsCidLength(Integer dtlsCidLength) throws Exception { + testPskDtlsCidLength(dtlsCidLength, serverDtlsCidLength); + } +} diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength3Test.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength1Test.java similarity index 78% rename from application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength3Test.java rename to application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength1Test.java index 8a65e28975..a36a618bcf 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength3Test.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength1Test.java @@ -21,17 +21,20 @@ import org.thingsboard.server.dao.service.DaoSqlTest; @TestPropertySource(properties = { - "transport.lwm2m.dtls.connection_id_length=3" + "transport.lwm2m.dtls.connection_id_length=1" }) @DaoSqlTest @Slf4j -public abstract class AbstractSecurityLwM2MIntegrationDtlsCidLength3Test extends AbstractSecurityLwM2MIntegrationDtlsCidLengthTest { +public abstract class AbstractSecurityLwM2MIntegrationDtlsCidLength1Test extends AbstractSecurityLwM2MIntegrationDtlsCidLengthTest { + + + private static final Integer serverDtlsCidLength = 1; protected void testNoSecDtlsCidLength(Integer dtlsCidLength) throws Exception { - testNoSecDtlsCidLength(dtlsCidLength, 3); + testNoSecDtlsCidLength(dtlsCidLength, serverDtlsCidLength); } protected void testPskDtlsCidLength(Integer dtlsCidLength) throws Exception { - testPskDtlsCidLength(dtlsCidLength, 3); + testPskDtlsCidLength(dtlsCidLength, serverDtlsCidLength); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength4Test.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength4Test.java new file mode 100644 index 0000000000..56e544243f --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength4Test.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.security.cid; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.dao.service.DaoSqlTest; + + +@TestPropertySource(properties = { + "transport.lwm2m.dtls.connection_id_length=4" +}) + +@DaoSqlTest +@Slf4j +public abstract class AbstractSecurityLwM2MIntegrationDtlsCidLength4Test extends AbstractSecurityLwM2MIntegrationDtlsCidLengthTest { + + private static final Integer serverDtlsCidLength = 4; + + protected void testNoSecDtlsCidLength(Integer dtlsCidLength) throws Exception { + testNoSecDtlsCidLength(dtlsCidLength, serverDtlsCidLength); + } + protected void testPskDtlsCidLength(Integer dtlsCidLength) throws Exception { + testPskDtlsCidLength(dtlsCidLength, serverDtlsCidLength); + } +} diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength8Test.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength8Test.java new file mode 100644 index 0000000000..341d8a8f86 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength8Test.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.security.cid; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.dao.service.DaoSqlTest; + + +@TestPropertySource(properties = { + "transport.lwm2m.dtls.connection_id_length=8" +}) + +@DaoSqlTest +@Slf4j +public abstract class AbstractSecurityLwM2MIntegrationDtlsCidLength8Test extends AbstractSecurityLwM2MIntegrationDtlsCidLengthTest { + + private static final Integer serverDtlsCidLength = 8; + + protected void testNoSecDtlsCidLength(Integer dtlsCidLength) throws Exception { + testNoSecDtlsCidLength(dtlsCidLength, serverDtlsCidLength); + } + protected void testPskDtlsCidLength(Integer dtlsCidLength) throws Exception { + testPskDtlsCidLength(dtlsCidLength, serverDtlsCidLength); + } +} diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLengthNullTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLengthNullTest.java index 9d52920072..6dea2f54d0 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLengthNullTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLengthNullTest.java @@ -28,11 +28,12 @@ import org.thingsboard.server.dao.service.DaoSqlTest; @Slf4j public abstract class AbstractSecurityLwM2MIntegrationDtlsCidLengthNullTest extends AbstractSecurityLwM2MIntegrationDtlsCidLengthTest { + private static final Integer serverDtlsCidLength = null; protected void testNoSecDtlsCidLength(Integer dtlsCidLength) throws Exception { - testNoSecDtlsCidLength(dtlsCidLength, null); + testNoSecDtlsCidLength(dtlsCidLength, serverDtlsCidLength); } protected void testPskDtlsCidLength(Integer dtlsCidLength) throws Exception { - testPskDtlsCidLength(dtlsCidLength, null); + testPskDtlsCidLength(dtlsCidLength, serverDtlsCidLength); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLengthTest.java index eb17f2cf7e..1ceab91153 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLengthTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLengthTest.java @@ -17,14 +17,29 @@ package org.thingsboard.server.transport.lwm2m.security.cid; import lombok.extern.slf4j.Slf4j; import org.eclipse.californium.elements.config.Configuration; +import org.eclipse.californium.scandium.DTLSConnector; +import org.eclipse.californium.scandium.dtls.Connection; +import org.eclipse.californium.scandium.dtls.ConnectionId; +import org.eclipse.californium.scandium.dtls.DTLSSession; +import org.eclipse.californium.scandium.dtls.InMemoryReadWriteLockConnectionStore; +import org.eclipse.californium.scandium.dtls.ResumptionSupportingConnectionStore; import org.eclipse.leshan.client.californium.endpoint.CaliforniumClientEndpoint; import org.eclipse.leshan.client.californium.endpoint.CaliforniumClientEndpointsProvider; +import org.eclipse.leshan.client.endpoint.LwM2mClientEndpoint; +import org.eclipse.leshan.client.servers.LwM2mServer; +import org.eclipse.leshan.core.peer.IpPeer; +import org.eclipse.leshan.core.peer.LwM2mPeer; import org.junit.Assert; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.transport.lwm2m.client.Lwm2mServer; import org.thingsboard.server.transport.lwm2m.security.AbstractSecurityLwM2MIntegrationTest; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.Collection; +import java.util.Collections; import java.util.concurrent.TimeUnit; import static org.awaitility.Awaitility.await; @@ -70,7 +85,6 @@ public abstract class AbstractSecurityLwM2MIntegrationDtlsCidLengthTest extends } else { Assert.assertEquals(2L, lwM2MTestClient.getClientDtlsCid().size()); Assert.assertTrue(lwM2MTestClient.getClientDtlsCid().keySet().contains(ON_READ_CONNECTION_ID)); - Assert.assertTrue(lwM2MTestClient.getClientDtlsCid().keySet().contains(ON_WRITE_CONNECTION_ID)); if (serverDtlsCidLength == null) { Assert.assertNull(lwM2MTestClient.getClientDtlsCid().get(ON_WRITE_CONNECTION_ID)); Assert.assertNull(lwM2MTestClient.getClientDtlsCid().get(ON_READ_CONNECTION_ID)); @@ -79,9 +93,30 @@ public abstract class AbstractSecurityLwM2MIntegrationDtlsCidLengthTest extends if (clientDtlsCidLength == null) { Assert.assertNull(lwM2MTestClient.getClientDtlsCid().get(ON_READ_CONNECTION_ID)); } else { - Assert.assertEquals(Integer.valueOf(serverDtlsCidLength), lwM2MTestClient.getClientDtlsCid().get(ON_WRITE_CONNECTION_ID)); + Integer expectedWrite = Math.max(clientDtlsCidLength, serverDtlsCidLength); +// Assert.assertEquals(expectedWrite, lwM2MTestClient.getClientDtlsCid().get(ON_WRITE_CONNECTION_ID)); + Assert.assertEquals(serverDtlsCidLength, lwM2MTestClient.getClientDtlsCid().get(ON_WRITE_CONNECTION_ID)); } } + LwM2mServer lwM2mServer = lwM2MTestClient.getLeshanClient().getRegisteredServers().entrySet().stream().findFirst().get().getValue(); + CaliforniumClientEndpoint lwM2mClientEndpoint = (CaliforniumClientEndpoint) lwM2MTestClient.getLeshanClient().getEndpoint(lwM2mServer); + DTLSConnector connector = (DTLSConnector) lwM2mClientEndpoint.getCoapEndpoint().getConnector(); + Field field = DTLSConnector.class.getDeclaredField("connectionStore"); + field.setAccessible(true); + ResumptionSupportingConnectionStore connectionStore = (InMemoryReadWriteLockConnectionStore) field.get(connector); + InetSocketAddress serverAddr = ((IpPeer) lwM2mServer.getTransportData()).getSocketAddress(); + Connection connection = connectionStore.get(serverAddr); + ConnectionId cid = connection.getConnectionId(); + if (cid != null) { + int actualClientCidLength = cid.getBytes().length; + int expectedClientCidLength; + if (clientDtlsCidLength == null || clientDtlsCidLength == 0) { + expectedClientCidLength = 3; + } else { + expectedClientCidLength = clientDtlsCidLength; + } + Assert.assertEquals(expectedClientCidLength, actualClientCidLength); + } } } } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_0/NoSecLwM2MIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_0/NoSecLwM2MIntegrationDtlsCidLengthTest.java index 6d529b3c08..f45ad5e86b 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_0/NoSecLwM2MIntegrationDtlsCidLengthTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_0/NoSecLwM2MIntegrationDtlsCidLengthTest.java @@ -41,7 +41,22 @@ public class NoSecLwM2MIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2 } @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_2() throws Exception { - testNoSecDtlsCidLength(2); + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_1() throws Exception { + testNoSecDtlsCidLength(1); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testNoSecDtlsCidLength(4); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { + testNoSecDtlsCidLength(8); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { + testNoSecDtlsCidLength(16); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_0/PskLwm2mIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_0/PskLwm2mIntegrationDtlsCidLengthTest.java index f478a18777..bf7effb336 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_0/PskLwm2mIntegrationDtlsCidLengthTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_0/PskLwm2mIntegrationDtlsCidLengthTest.java @@ -39,10 +39,23 @@ public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MI public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_0() throws Exception { testPskDtlsCidLength(0); } + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_1() throws Exception { + testPskDtlsCidLength(1); + } @Test - public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_2() throws Exception { - testPskDtlsCidLength(2); + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testPskDtlsCidLength(4); + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { + testPskDtlsCidLength(8); + } + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { + testPskDtlsCidLength(16); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_1/NoSecLwM2MIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_1/NoSecLwM2MIntegrationDtlsCidLengthTest.java new file mode 100644 index 0000000000..6f9a2ac753 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_1/NoSecLwM2MIntegrationDtlsCidLengthTest.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.security.cid.serverDtlsCidLength_1; + +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength0Test; +import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength1Test; + +import static org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode.NO_SEC; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE; + +public class NoSecLwM2MIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MIntegrationDtlsCidLength1Test { + + @Before + public void setUpNoSecDtlsCidLength() { + transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(NO_SEC, NONE)); + awaitAlias = "await on client state (NoSec_Lwm2m) DtlsCidLength = 1"; + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_Null() throws Exception { + testNoSecDtlsCidLength(null); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_0() throws Exception { + testNoSecDtlsCidLength(0); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_1() throws Exception { + testNoSecDtlsCidLength(1); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testNoSecDtlsCidLength(4); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { + testNoSecDtlsCidLength(8); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { + testNoSecDtlsCidLength(16); + } +} diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_1/PskLwm2mIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_1/PskLwm2mIntegrationDtlsCidLengthTest.java new file mode 100644 index 0000000000..846c993b3e --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_1/PskLwm2mIntegrationDtlsCidLengthTest.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.security.cid.serverDtlsCidLength_1; + +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength0Test; +import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength1Test; + +import static org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode.PSK; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE; + +public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MIntegrationDtlsCidLength1Test { + + @Before + public void createProfileRpc() { + transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(PSK, NONE)); + awaitAlias = "await on client state (Psk_Lwm2m) DtlsCidLength = 1"; + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_Null() throws Exception { + testPskDtlsCidLength(null); + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_0() throws Exception { + testPskDtlsCidLength(0); + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_1() throws Exception { + testPskDtlsCidLength(1); + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testPskDtlsCidLength(4); + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { + testPskDtlsCidLength(8); + } + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { + testPskDtlsCidLength(16); + } +} + diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_16/NoSecLwM2MIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_16/NoSecLwM2MIntegrationDtlsCidLengthTest.java new file mode 100644 index 0000000000..7b54b83d3e --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_16/NoSecLwM2MIntegrationDtlsCidLengthTest.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.security.cid.serverDtlsCidLength_16; + +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength16Test; +import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength4Test; + +import static org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode.NO_SEC; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE; + +public class NoSecLwM2MIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MIntegrationDtlsCidLength16Test { + + @Before + public void setUpNoSecDtlsCidLength() { + transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(NO_SEC, NONE)); + awaitAlias = "await on client state (NoSec_Lwm2m) DtlsCidLength = 16"; + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_Null() throws Exception { + testNoSecDtlsCidLength(null); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_0() throws Exception { + testNoSecDtlsCidLength(0); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_1() throws Exception { + testNoSecDtlsCidLength(1); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testNoSecDtlsCidLength(4); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { + testNoSecDtlsCidLength(8); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { + testNoSecDtlsCidLength(16); + } +} diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_16/PskLwm2mIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_16/PskLwm2mIntegrationDtlsCidLengthTest.java new file mode 100644 index 0000000000..21c4428510 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_16/PskLwm2mIntegrationDtlsCidLengthTest.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.security.cid.serverDtlsCidLength_16; + +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength16Test; +import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength4Test; + +import static org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode.PSK; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE; + +public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MIntegrationDtlsCidLength16Test { + + @Before + public void createProfileRpc() { + transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(PSK, NONE)); + awaitAlias = "await on client state (Psk_Lwm2m) DtlsCidLength = 16"; + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_Null() throws Exception { + testPskDtlsCidLength(null); + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_0() throws Exception { + testPskDtlsCidLength(0); + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_1() throws Exception { + testPskDtlsCidLength(1); + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testPskDtlsCidLength(4); + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { + testPskDtlsCidLength(8); + } + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { + testPskDtlsCidLength(16); + } +} + diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_3/NoSecLwM2MIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_4/NoSecLwM2MIntegrationDtlsCidLengthTest.java similarity index 73% rename from application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_3/NoSecLwM2MIntegrationDtlsCidLengthTest.java rename to application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_4/NoSecLwM2MIntegrationDtlsCidLengthTest.java index a395f2e7e3..258e0d1263 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_3/NoSecLwM2MIntegrationDtlsCidLengthTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_4/NoSecLwM2MIntegrationDtlsCidLengthTest.java @@ -13,21 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.transport.lwm2m.security.cid.serverDtlsCidLength_3; +package org.thingsboard.server.transport.lwm2m.security.cid.serverDtlsCidLength_4; import org.junit.Before; import org.junit.Test; -import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength3Test; +import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength4Test; import static org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode.NO_SEC; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE; -public class NoSecLwM2MIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MIntegrationDtlsCidLength3Test { +public class NoSecLwM2MIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MIntegrationDtlsCidLength4Test { @Before public void setUpNoSecDtlsCidLength() { transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(NO_SEC, NONE)); - awaitAlias = "await on client state (NoSec_Lwm2m) DtlsCidLength = 3"; + awaitAlias = "await on client state (NoSec_Lwm2m) DtlsCidLength = 4"; } @Test @@ -41,7 +41,22 @@ public class NoSecLwM2MIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2 } @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_2() throws Exception { - testNoSecDtlsCidLength(2); + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_1() throws Exception { + testNoSecDtlsCidLength(1); } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testNoSecDtlsCidLength(4); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { + testNoSecDtlsCidLength(8); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { + testNoSecDtlsCidLength(16); + } } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_3/PskLwm2mIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_4/PskLwm2mIntegrationDtlsCidLengthTest.java similarity index 73% rename from application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_3/PskLwm2mIntegrationDtlsCidLengthTest.java rename to application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_4/PskLwm2mIntegrationDtlsCidLengthTest.java index 868a146ed7..4c08d5e0cc 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_3/PskLwm2mIntegrationDtlsCidLengthTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_4/PskLwm2mIntegrationDtlsCidLengthTest.java @@ -13,21 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.transport.lwm2m.security.cid.serverDtlsCidLength_3; +package org.thingsboard.server.transport.lwm2m.security.cid.serverDtlsCidLength_4; import org.junit.Before; import org.junit.Test; -import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength3Test; +import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength4Test; import static org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode.PSK; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE; -public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MIntegrationDtlsCidLength3Test { +public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MIntegrationDtlsCidLength4Test { @Before public void createProfileRpc() { transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(PSK, NONE)); - awaitAlias = "await on client state (Psk_Lwm2m) DtlsCidLength = 3"; + awaitAlias = "await on client state (Psk_Lwm2m) DtlsCidLength = 4"; } @Test @@ -41,8 +41,22 @@ public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MI } @Test - public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_2() throws Exception { - testPskDtlsCidLength(2); + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_1() throws Exception { + testPskDtlsCidLength(1); + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testPskDtlsCidLength(4); + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { + testPskDtlsCidLength(8); + } + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { + testPskDtlsCidLength(16); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_8/NoSecLwM2MIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_8/NoSecLwM2MIntegrationDtlsCidLengthTest.java new file mode 100644 index 0000000000..7af7a73544 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_8/NoSecLwM2MIntegrationDtlsCidLengthTest.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.security.cid.serverDtlsCidLength_8; + +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength4Test; +import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength8Test; + +import static org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode.NO_SEC; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE; + +public class NoSecLwM2MIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MIntegrationDtlsCidLength8Test { + + @Before + public void setUpNoSecDtlsCidLength() { + transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(NO_SEC, NONE)); + awaitAlias = "await on client state (NoSec_Lwm2m) DtlsCidLength = 8"; + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_Null() throws Exception { + testNoSecDtlsCidLength(null); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_0() throws Exception { + testNoSecDtlsCidLength(0); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_1() throws Exception { + testNoSecDtlsCidLength(1); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testNoSecDtlsCidLength(4); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { + testNoSecDtlsCidLength(8); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { + testNoSecDtlsCidLength(16); + } +} diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_8/PskLwm2mIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_8/PskLwm2mIntegrationDtlsCidLengthTest.java new file mode 100644 index 0000000000..32e6802375 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_8/PskLwm2mIntegrationDtlsCidLengthTest.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.security.cid.serverDtlsCidLength_8; + +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength4Test; +import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength8Test; + +import static org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode.PSK; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE; + +public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MIntegrationDtlsCidLength8Test { + + @Before + public void createProfileRpc() { + transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(PSK, NONE)); + awaitAlias = "await on client state (Psk_Lwm2m) DtlsCidLength = 8"; + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_Null() throws Exception { + testPskDtlsCidLength(null); + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_0() throws Exception { + testPskDtlsCidLength(0); + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_1() throws Exception { + testPskDtlsCidLength(1); + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testPskDtlsCidLength(4); + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { + testPskDtlsCidLength(8); + } + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { + testPskDtlsCidLength(16); + } +} + diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_null/NoSecLwM2MIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_null/NoSecLwM2MIntegrationDtlsCidLengthTest.java index 9e7424743a..433029a4ca 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_null/NoSecLwM2MIntegrationDtlsCidLengthTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_null/NoSecLwM2MIntegrationDtlsCidLengthTest.java @@ -41,7 +41,22 @@ public class NoSecLwM2MIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2 } @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_2() throws Exception { - testNoSecDtlsCidLength(2); + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_1() throws Exception { + testNoSecDtlsCidLength(1); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testNoSecDtlsCidLength(4); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { + testNoSecDtlsCidLength(8); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { + testNoSecDtlsCidLength(16); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_null/PskLwm2mIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_null/PskLwm2mIntegrationDtlsCidLengthTest.java index 8a8f01b3ab..ea95dca059 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_null/PskLwm2mIntegrationDtlsCidLengthTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_null/PskLwm2mIntegrationDtlsCidLengthTest.java @@ -41,8 +41,22 @@ public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MI } @Test - public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_2() throws Exception { - testPskDtlsCidLength(2); + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_1() throws Exception { + testPskDtlsCidLength(1); + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testPskDtlsCidLength(4); + } + + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { + testPskDtlsCidLength(8); + } + @Test + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { + testPskDtlsCidLength(16); } } From 0227b2b314009545fe5c52dfbee7a3d087d434e0 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Tue, 11 Nov 2025 09:41:11 +0200 Subject: [PATCH 27/40] lwm2m: test CID client - 2 --- ...tyLwM2MIntegrationDtlsCidLength2Test.java} | 6 +- ...oSecLwM2MIntegrationDtlsCidLengthTest.java | 63 ------------------- .../PskLwm2mIntegrationDtlsCidLengthTest.java | 18 +++--- ...oSecLwM2MIntegrationDtlsCidLengthTest.java | 62 ------------------ ...oSecLwM2MIntegrationDtlsCidLengthTest.java | 63 ------------------- 5 files changed, 12 insertions(+), 200 deletions(-) rename application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/{AbstractSecurityLwM2MIntegrationDtlsCidLength8Test.java => AbstractSecurityLwM2MIntegrationDtlsCidLength2Test.java} (87%) delete mode 100644 application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_1/NoSecLwM2MIntegrationDtlsCidLengthTest.java rename application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/{serverDtlsCidLength_8 => serverDtlsCidLength_2}/PskLwm2mIntegrationDtlsCidLengthTest.java (87%) delete mode 100644 application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_4/NoSecLwM2MIntegrationDtlsCidLengthTest.java delete mode 100644 application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_8/NoSecLwM2MIntegrationDtlsCidLengthTest.java diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength8Test.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength2Test.java similarity index 87% rename from application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength8Test.java rename to application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength2Test.java index 341d8a8f86..1cb657e4a4 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength8Test.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength2Test.java @@ -21,14 +21,14 @@ import org.thingsboard.server.dao.service.DaoSqlTest; @TestPropertySource(properties = { - "transport.lwm2m.dtls.connection_id_length=8" + "transport.lwm2m.dtls.connection_id_length=2" }) @DaoSqlTest @Slf4j -public abstract class AbstractSecurityLwM2MIntegrationDtlsCidLength8Test extends AbstractSecurityLwM2MIntegrationDtlsCidLengthTest { +public abstract class AbstractSecurityLwM2MIntegrationDtlsCidLength2Test extends AbstractSecurityLwM2MIntegrationDtlsCidLengthTest { - private static final Integer serverDtlsCidLength = 8; + private static final Integer serverDtlsCidLength = 2; protected void testNoSecDtlsCidLength(Integer dtlsCidLength) throws Exception { testNoSecDtlsCidLength(dtlsCidLength, serverDtlsCidLength); diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_1/NoSecLwM2MIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_1/NoSecLwM2MIntegrationDtlsCidLengthTest.java deleted file mode 100644 index 6f9a2ac753..0000000000 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_1/NoSecLwM2MIntegrationDtlsCidLengthTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.transport.lwm2m.security.cid.serverDtlsCidLength_1; - -import org.junit.Before; -import org.junit.Test; -import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength0Test; -import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength1Test; - -import static org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode.NO_SEC; -import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE; - -public class NoSecLwM2MIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MIntegrationDtlsCidLength1Test { - - @Before - public void setUpNoSecDtlsCidLength() { - transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(NO_SEC, NONE)); - awaitAlias = "await on client state (NoSec_Lwm2m) DtlsCidLength = 1"; - } - - @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_Null() throws Exception { - testNoSecDtlsCidLength(null); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_0() throws Exception { - testNoSecDtlsCidLength(0); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_1() throws Exception { - testNoSecDtlsCidLength(1); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { - testNoSecDtlsCidLength(4); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { - testNoSecDtlsCidLength(8); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { - testNoSecDtlsCidLength(16); - } -} diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_8/PskLwm2mIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_2/PskLwm2mIntegrationDtlsCidLengthTest.java similarity index 87% rename from application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_8/PskLwm2mIntegrationDtlsCidLengthTest.java rename to application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_2/PskLwm2mIntegrationDtlsCidLengthTest.java index 32e6802375..a1a1d18042 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_8/PskLwm2mIntegrationDtlsCidLengthTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_2/PskLwm2mIntegrationDtlsCidLengthTest.java @@ -13,22 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.transport.lwm2m.security.cid.serverDtlsCidLength_8; +package org.thingsboard.server.transport.lwm2m.security.cid.serverDtlsCidLength_2; import org.junit.Before; import org.junit.Test; -import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength4Test; -import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength8Test; +import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength2Test; import static org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode.PSK; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE; -public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MIntegrationDtlsCidLength8Test { +public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MIntegrationDtlsCidLength2Test { @Before public void createProfileRpc() { transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(PSK, NONE)); - awaitAlias = "await on client state (Psk_Lwm2m) DtlsCidLength = 8"; + awaitAlias = "await on client state (Psk_Lwm2m) DtlsCidLength = 2"; } @Test @@ -47,14 +46,15 @@ public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MI } @Test - public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { - testPskDtlsCidLength(4); + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_2() throws Exception { + testPskDtlsCidLength(2); } @Test - public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { - testPskDtlsCidLength(8); + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testPskDtlsCidLength(4); } + @Test public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { testPskDtlsCidLength(16); diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_4/NoSecLwM2MIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_4/NoSecLwM2MIntegrationDtlsCidLengthTest.java deleted file mode 100644 index 258e0d1263..0000000000 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_4/NoSecLwM2MIntegrationDtlsCidLengthTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.transport.lwm2m.security.cid.serverDtlsCidLength_4; - -import org.junit.Before; -import org.junit.Test; -import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength4Test; - -import static org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode.NO_SEC; -import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE; - -public class NoSecLwM2MIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MIntegrationDtlsCidLength4Test { - - @Before - public void setUpNoSecDtlsCidLength() { - transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(NO_SEC, NONE)); - awaitAlias = "await on client state (NoSec_Lwm2m) DtlsCidLength = 4"; - } - - @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_Null() throws Exception { - testNoSecDtlsCidLength(null); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_0() throws Exception { - testNoSecDtlsCidLength(0); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_1() throws Exception { - testNoSecDtlsCidLength(1); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { - testNoSecDtlsCidLength(4); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { - testNoSecDtlsCidLength(8); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { - testNoSecDtlsCidLength(16); - } -} diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_8/NoSecLwM2MIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_8/NoSecLwM2MIntegrationDtlsCidLengthTest.java deleted file mode 100644 index 7af7a73544..0000000000 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_8/NoSecLwM2MIntegrationDtlsCidLengthTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.transport.lwm2m.security.cid.serverDtlsCidLength_8; - -import org.junit.Before; -import org.junit.Test; -import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength4Test; -import org.thingsboard.server.transport.lwm2m.security.cid.AbstractSecurityLwM2MIntegrationDtlsCidLength8Test; - -import static org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode.NO_SEC; -import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE; - -public class NoSecLwM2MIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MIntegrationDtlsCidLength8Test { - - @Before - public void setUpNoSecDtlsCidLength() { - transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(NO_SEC, NONE)); - awaitAlias = "await on client state (NoSec_Lwm2m) DtlsCidLength = 8"; - } - - @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_Null() throws Exception { - testNoSecDtlsCidLength(null); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_0() throws Exception { - testNoSecDtlsCidLength(0); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_1() throws Exception { - testNoSecDtlsCidLength(1); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { - testNoSecDtlsCidLength(4); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { - testNoSecDtlsCidLength(8); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { - testNoSecDtlsCidLength(16); - } -} From 9e11e21bb89b37a20907a88953f89130ede5ca74 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Tue, 11 Nov 2025 15:43:12 +0200 Subject: [PATCH 28/40] lwm2m: finish test CID client --- ...tyLwM2MIntegrationDtlsCidLength16Test.java | 9 +-- ...rityLwM2MIntegrationDtlsCidLengthTest.java | 68 +++++++++++-------- ...oSecLwM2MIntegrationDtlsCidLengthTest.java | 10 +-- .../PskLwm2mIntegrationDtlsCidLengthTest.java | 11 +-- .../PskLwm2mIntegrationDtlsCidLengthTest.java | 11 +-- ...oSecLwM2MIntegrationDtlsCidLengthTest.java | 10 +-- .../PskLwm2mIntegrationDtlsCidLengthTest.java | 11 +-- .../PskLwm2mIntegrationDtlsCidLengthTest.java | 2 +- .../PskLwm2mIntegrationDtlsCidLengthTest.java | 11 +-- ...oSecLwM2MIntegrationDtlsCidLengthTest.java | 2 +- .../PskLwm2mIntegrationDtlsCidLengthTest.java | 11 +-- 11 files changed, 86 insertions(+), 70 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength16Test.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength16Test.java index 1481153d8d..1f5d2d9f19 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength16Test.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLength16Test.java @@ -30,10 +30,11 @@ public abstract class AbstractSecurityLwM2MIntegrationDtlsCidLength16Test extend private static final Integer serverDtlsCidLength = 16; - protected void testNoSecDtlsCidLength(Integer dtlsCidLength) throws Exception { - testNoSecDtlsCidLength(dtlsCidLength, serverDtlsCidLength); + protected void testNoSecDtlsCidLength(Integer clientDtlsCidLength) throws Exception { + testNoSecDtlsCidLength(clientDtlsCidLength, serverDtlsCidLength); } - protected void testPskDtlsCidLength(Integer dtlsCidLength) throws Exception { - testPskDtlsCidLength(dtlsCidLength, serverDtlsCidLength); + + protected void testPskDtlsCidLength(Integer clientDtlsCidLength) throws Exception { + testPskDtlsCidLength(clientDtlsCidLength, serverDtlsCidLength); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLengthTest.java index 1ceab91153..04afef0c61 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLengthTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/AbstractSecurityLwM2MIntegrationDtlsCidLengthTest.java @@ -20,26 +20,20 @@ import org.eclipse.californium.elements.config.Configuration; import org.eclipse.californium.scandium.DTLSConnector; import org.eclipse.californium.scandium.dtls.Connection; import org.eclipse.californium.scandium.dtls.ConnectionId; -import org.eclipse.californium.scandium.dtls.DTLSSession; import org.eclipse.californium.scandium.dtls.InMemoryReadWriteLockConnectionStore; import org.eclipse.californium.scandium.dtls.ResumptionSupportingConnectionStore; import org.eclipse.leshan.client.californium.endpoint.CaliforniumClientEndpoint; import org.eclipse.leshan.client.californium.endpoint.CaliforniumClientEndpointsProvider; -import org.eclipse.leshan.client.endpoint.LwM2mClientEndpoint; import org.eclipse.leshan.client.servers.LwM2mServer; import org.eclipse.leshan.core.peer.IpPeer; -import org.eclipse.leshan.core.peer.LwM2mPeer; import org.junit.Assert; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.transport.lwm2m.client.Lwm2mServer; import org.thingsboard.server.transport.lwm2m.security.AbstractSecurityLwM2MIntegrationTest; import java.lang.reflect.Field; import java.net.InetSocketAddress; -import java.util.Collection; -import java.util.Collections; import java.util.concurrent.TimeUnit; import static org.awaitility.Awaitility.await; @@ -54,13 +48,13 @@ public abstract class AbstractSecurityLwM2MIntegrationDtlsCidLengthTest extends protected String awaitAlias; - protected void testNoSecDtlsCidLength(Integer dtlsCidLength, Integer serverDtlsCidLength) throws Exception { + protected void testNoSecDtlsCidLength(Integer clientDtlsCidLength, Integer serverDtlsCidLength) throws Exception { initDeviceCredentialsNoSek(); - basicTestConnectionDtlsCidLength(dtlsCidLength, serverDtlsCidLength); + basicTestConnectionDtlsCidLength(clientDtlsCidLength, serverDtlsCidLength); } - protected void testPskDtlsCidLength(Integer dtlsCidLength, Integer serverDtlsCidLength) throws Exception { + protected void testPskDtlsCidLength(Integer clientDtlsCidLength, Integer serverDtlsCidLength) throws Exception { initDeviceCredentialsPsk(); - basicTestConnectionDtlsCidLength(dtlsCidLength, serverDtlsCidLength); + basicTestConnectionDtlsCidLength(clientDtlsCidLength, serverDtlsCidLength); } protected void basicTestConnectionDtlsCidLength(Integer clientDtlsCidLength, @@ -84,31 +78,38 @@ public abstract class AbstractSecurityLwM2MIntegrationDtlsCidLengthTest extends Assert.assertTrue(lwM2MTestClient.getClientDtlsCid().isEmpty()); } else { Assert.assertEquals(2L, lwM2MTestClient.getClientDtlsCid().size()); - Assert.assertTrue(lwM2MTestClient.getClientDtlsCid().keySet().contains(ON_READ_CONNECTION_ID)); - if (serverDtlsCidLength == null) { + Assert.assertTrue(lwM2MTestClient.getClientDtlsCid().containsKey(ON_READ_CONNECTION_ID)); + Assert.assertTrue(lwM2MTestClient.getClientDtlsCid().containsKey(ON_WRITE_CONNECTION_ID)); + + LwM2mServer lwM2mServer = lwM2MTestClient.getLeshanClient().getRegisteredServers().entrySet().stream().findFirst().get().getValue(); + CaliforniumClientEndpoint lwM2mClientEndpoint = (CaliforniumClientEndpoint) lwM2MTestClient.getLeshanClient().getEndpoint(lwM2mServer); + Connection connection = getConnection(lwM2mClientEndpoint, lwM2mServer); + ConnectionId clientCid = connection.getConnectionId(); + ConnectionId readCid = connection.getEstablishedDtlsContext().getReadConnectionId(); + ConnectionId serverCid = connection.getEstablishedDtlsContext().getWriteConnectionId(); + if (serverDtlsCidLength == null || clientDtlsCidLength == null) { + // cid is not used Assert.assertNull(lwM2MTestClient.getClientDtlsCid().get(ON_WRITE_CONNECTION_ID)); Assert.assertNull(lwM2MTestClient.getClientDtlsCid().get(ON_READ_CONNECTION_ID)); + Assert.assertNull(readCid); + Assert.assertNull(serverCid); } else { + Assert.assertEquals(serverDtlsCidLength, lwM2MTestClient.getClientDtlsCid().get(ON_WRITE_CONNECTION_ID)); Assert.assertEquals(clientDtlsCidLength, lwM2MTestClient.getClientDtlsCid().get(ON_READ_CONNECTION_ID)); - if (clientDtlsCidLength == null) { - Assert.assertNull(lwM2MTestClient.getClientDtlsCid().get(ON_READ_CONNECTION_ID)); - } else { - Integer expectedWrite = Math.max(clientDtlsCidLength, serverDtlsCidLength); -// Assert.assertEquals(expectedWrite, lwM2MTestClient.getClientDtlsCid().get(ON_WRITE_CONNECTION_ID)); - Assert.assertEquals(serverDtlsCidLength, lwM2MTestClient.getClientDtlsCid().get(ON_WRITE_CONNECTION_ID)); + // cid used + Assert.assertNotNull(clientCid); + Assert.assertNotNull(readCid); + if (clientDtlsCidLength > 0) { + Assert.assertEquals(clientCid, readCid); } + Assert.assertNotNull(serverCid); + int actualServerCidLength = serverCid.getBytes().length; + int expectedServerCidLength = serverDtlsCidLength; + Assert.assertEquals(expectedServerCidLength, actualServerCidLength); } - LwM2mServer lwM2mServer = lwM2MTestClient.getLeshanClient().getRegisteredServers().entrySet().stream().findFirst().get().getValue(); - CaliforniumClientEndpoint lwM2mClientEndpoint = (CaliforniumClientEndpoint) lwM2MTestClient.getLeshanClient().getEndpoint(lwM2mServer); - DTLSConnector connector = (DTLSConnector) lwM2mClientEndpoint.getCoapEndpoint().getConnector(); - Field field = DTLSConnector.class.getDeclaredField("connectionStore"); - field.setAccessible(true); - ResumptionSupportingConnectionStore connectionStore = (InMemoryReadWriteLockConnectionStore) field.get(connector); - InetSocketAddress serverAddr = ((IpPeer) lwM2mServer.getTransportData()).getSocketAddress(); - Connection connection = connectionStore.get(serverAddr); - ConnectionId cid = connection.getConnectionId(); - if (cid != null) { - int actualClientCidLength = cid.getBytes().length; + + if (clientCid != null) { + int actualClientCidLength = clientCid.getBytes().length; int expectedClientCidLength; if (clientDtlsCidLength == null || clientDtlsCidLength == 0) { expectedClientCidLength = 3; @@ -119,4 +120,13 @@ public abstract class AbstractSecurityLwM2MIntegrationDtlsCidLengthTest extends } } } + + private static Connection getConnection(CaliforniumClientEndpoint lwM2mClientEndpoint, LwM2mServer lwM2mServer) throws NoSuchFieldException, IllegalAccessException { + DTLSConnector connector = (DTLSConnector) lwM2mClientEndpoint.getCoapEndpoint().getConnector(); + Field field = DTLSConnector.class.getDeclaredField("connectionStore"); + field.setAccessible(true); + ResumptionSupportingConnectionStore connectionStore = (InMemoryReadWriteLockConnectionStore) field.get(connector); + InetSocketAddress serverAddr = ((IpPeer) lwM2mServer.getTransportData()).getSocketAddress(); + return connectionStore.get(serverAddr); + } } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_0/NoSecLwM2MIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_0/NoSecLwM2MIntegrationDtlsCidLengthTest.java index f45ad5e86b..2b5cccd7be 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_0/NoSecLwM2MIntegrationDtlsCidLengthTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_0/NoSecLwM2MIntegrationDtlsCidLengthTest.java @@ -27,7 +27,7 @@ public class NoSecLwM2MIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2 @Before public void setUpNoSecDtlsCidLength() { transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(NO_SEC, NONE)); - awaitAlias = "await on client state (NoSec_Lwm2m) DtlsCidLength = 0"; + awaitAlias = "await on client state (NoSec_Lwm2m) serverDtlsCidLength = 0"; } @Test @@ -46,13 +46,13 @@ public class NoSecLwM2MIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2 } @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { - testNoSecDtlsCidLength(4); + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_2() throws Exception { + testNoSecDtlsCidLength(1); } @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { - testNoSecDtlsCidLength(8); + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testNoSecDtlsCidLength(4); } @Test diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_0/PskLwm2mIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_0/PskLwm2mIntegrationDtlsCidLengthTest.java index bf7effb336..c08eeeaa79 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_0/PskLwm2mIntegrationDtlsCidLengthTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_0/PskLwm2mIntegrationDtlsCidLengthTest.java @@ -27,7 +27,7 @@ public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MI @Before public void createProfileRpc() { transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(PSK, NONE)); - awaitAlias = "await on client state (Psk_Lwm2m) DtlsCidLength = 0"; + awaitAlias = "await on client state (Psk_Lwm2m) serverDtlsCidLength = 0"; } @Test @@ -45,14 +45,15 @@ public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MI } @Test - public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { - testPskDtlsCidLength(4); + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_2() throws Exception { + testPskDtlsCidLength(2); } @Test - public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { - testPskDtlsCidLength(8); + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testPskDtlsCidLength(4); } + @Test public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { testPskDtlsCidLength(16); diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_1/PskLwm2mIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_1/PskLwm2mIntegrationDtlsCidLengthTest.java index 846c993b3e..b2c06495fc 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_1/PskLwm2mIntegrationDtlsCidLengthTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_1/PskLwm2mIntegrationDtlsCidLengthTest.java @@ -28,7 +28,7 @@ public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MI @Before public void createProfileRpc() { transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(PSK, NONE)); - awaitAlias = "await on client state (Psk_Lwm2m) DtlsCidLength = 1"; + awaitAlias = "await on client state (Psk_Lwm2m) serverDtlsCidLength = 1"; } @Test @@ -47,14 +47,15 @@ public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MI } @Test - public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { - testPskDtlsCidLength(4); + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_2() throws Exception { + testPskDtlsCidLength(2); } @Test - public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { - testPskDtlsCidLength(8); + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testPskDtlsCidLength(4); } + @Test public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { testPskDtlsCidLength(16); diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_16/NoSecLwM2MIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_16/NoSecLwM2MIntegrationDtlsCidLengthTest.java index 7b54b83d3e..872608145c 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_16/NoSecLwM2MIntegrationDtlsCidLengthTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_16/NoSecLwM2MIntegrationDtlsCidLengthTest.java @@ -28,7 +28,7 @@ public class NoSecLwM2MIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2 @Before public void setUpNoSecDtlsCidLength() { transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(NO_SEC, NONE)); - awaitAlias = "await on client state (NoSec_Lwm2m) DtlsCidLength = 16"; + awaitAlias = "await on client state (NoSec_Lwm2m) serverDtlsCidLength = 16"; } @Test @@ -47,13 +47,13 @@ public class NoSecLwM2MIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2 } @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { - testNoSecDtlsCidLength(4); + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_2() throws Exception { + testNoSecDtlsCidLength(2); } @Test - public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { - testNoSecDtlsCidLength(8); + public void testWithNoSecConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testNoSecDtlsCidLength(4); } @Test diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_16/PskLwm2mIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_16/PskLwm2mIntegrationDtlsCidLengthTest.java index 21c4428510..579614d98e 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_16/PskLwm2mIntegrationDtlsCidLengthTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_16/PskLwm2mIntegrationDtlsCidLengthTest.java @@ -28,7 +28,7 @@ public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MI @Before public void createProfileRpc() { transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(PSK, NONE)); - awaitAlias = "await on client state (Psk_Lwm2m) DtlsCidLength = 16"; + awaitAlias = "await on client state (Psk_Lwm2m) serverDtlsCidLength = 16"; } @Test @@ -47,14 +47,15 @@ public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MI } @Test - public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { - testPskDtlsCidLength(4); + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_2() throws Exception { + testPskDtlsCidLength(2); } @Test - public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { - testPskDtlsCidLength(8); + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testPskDtlsCidLength(4); } + @Test public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { testPskDtlsCidLength(16); diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_2/PskLwm2mIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_2/PskLwm2mIntegrationDtlsCidLengthTest.java index a1a1d18042..2d68d057d8 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_2/PskLwm2mIntegrationDtlsCidLengthTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_2/PskLwm2mIntegrationDtlsCidLengthTest.java @@ -27,7 +27,7 @@ public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MI @Before public void createProfileRpc() { transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(PSK, NONE)); - awaitAlias = "await on client state (Psk_Lwm2m) DtlsCidLength = 2"; + awaitAlias = "await on client state (Psk_Lwm2m) serverDtlsCidLength = 2"; } @Test diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_4/PskLwm2mIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_4/PskLwm2mIntegrationDtlsCidLengthTest.java index 4c08d5e0cc..6994e19fbf 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_4/PskLwm2mIntegrationDtlsCidLengthTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_4/PskLwm2mIntegrationDtlsCidLengthTest.java @@ -27,7 +27,7 @@ public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MI @Before public void createProfileRpc() { transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(PSK, NONE)); - awaitAlias = "await on client state (Psk_Lwm2m) DtlsCidLength = 4"; + awaitAlias = "await on client state (Psk_Lwm2m) serverDtlsCidLength = 4"; } @Test @@ -46,14 +46,15 @@ public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MI } @Test - public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { - testPskDtlsCidLength(4); + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_2() throws Exception { + testPskDtlsCidLength(2); } @Test - public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { - testPskDtlsCidLength(8); + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testPskDtlsCidLength(4); } + @Test public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { testPskDtlsCidLength(16); diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_null/NoSecLwM2MIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_null/NoSecLwM2MIntegrationDtlsCidLengthTest.java index 433029a4ca..e85e03dfed 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_null/NoSecLwM2MIntegrationDtlsCidLengthTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_null/NoSecLwM2MIntegrationDtlsCidLengthTest.java @@ -27,7 +27,7 @@ public class NoSecLwM2MIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2 @Before public void setUpNoSecDtlsCidLength() { transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(NO_SEC, NONE)); - awaitAlias = "await on client state (NoSec_Lwm2m) DtlsCidLength = Null"; + awaitAlias = "await on client state (NoSec_Lwm2m) serverDtlsCidLength = Null"; } @Test diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_null/PskLwm2mIntegrationDtlsCidLengthTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_null/PskLwm2mIntegrationDtlsCidLengthTest.java index ea95dca059..e482e1b106 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_null/PskLwm2mIntegrationDtlsCidLengthTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/cid/serverDtlsCidLength_null/PskLwm2mIntegrationDtlsCidLengthTest.java @@ -27,7 +27,7 @@ public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MI @Before public void createProfileRpc() { transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(PSK, NONE)); - awaitAlias = "await on client state (Psk_Lwm2m) DtlsCidLength = Null"; + awaitAlias = "await on client state (Psk_Lwm2m) serverDtlsCidLength = Null"; } @Test @@ -46,14 +46,15 @@ public class PskLwm2mIntegrationDtlsCidLengthTest extends AbstractSecurityLwM2MI } @Test - public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { - testPskDtlsCidLength(4); + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_2() throws Exception { + testPskDtlsCidLength(2); } @Test - public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_8() throws Exception { - testPskDtlsCidLength(8); + public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_4() throws Exception { + testPskDtlsCidLength(4); } + @Test public void testWithPskConnectLwm2mSuccessClientDtlsCidLength_16() throws Exception { testPskDtlsCidLength(16); From c03085bbe7c46b568bf3cdc7c83f8a32b9c0f25e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 11 Nov 2025 16:15:42 +0200 Subject: [PATCH 29/40] updated init method to handle restore events --- .../main/data/upgrade/basic/schema_update.sql | 6 ++--- .../server/actors/ActorSystemContext.java | 10 +++++--- ...CalculatedFieldEntityMessageProcessor.java | 14 ++++------- .../controller/SystemInfoController.java | 2 +- .../controller/TenantProfileController.java | 5 ++++ ...tractCalculatedFieldProcessingService.java | 9 +++---- .../service/cf/CalculatedFieldCache.java | 6 ++--- .../cf/DefaultCalculatedFieldCache.java | 18 ++++---------- .../DefaultCalculatedFieldQueueService.java | 11 +++++---- .../service/cf/ctx/state/ArgumentEntry.java | 6 +++++ .../ctx/state/BaseCalculatedFieldState.java | 2 +- .../cf/ctx/state/CalculatedFieldCtx.java | 24 ++++++++++++++----- .../cf/ctx/state/CalculatedFieldState.java | 4 ++-- ...titiesAggregationCalculatedFieldState.java | 8 +++++++ .../aggregation/single/AggIntervalEntry.java | 2 +- .../EntityAggregationArgumentEntry.java | 7 ++++++ ...EntityAggregationCalculatedFieldState.java | 16 +++++++++---- .../alarm/AlarmCalculatedFieldState.java | 7 ++++-- .../src/main/resources/thingsboard.yml | 3 +++ .../GeofencingCalculatedFieldStateTest.java | 2 +- .../PropagationCalculatedFieldStateTest.java | 2 +- .../state/ScriptCalculatedFieldStateTest.java | 2 +- .../state/SimpleCalculatedFieldStateTest.java | 2 +- .../server/common/data/SystemParams.java | 2 +- ...gregationCalculatedFieldConfiguration.java | 6 +++++ .../single/interval/BaseAggInterval.java | 4 ++++ .../single/interval/Watermark.java | 2 ++ .../DefaultTenantProfileConfiguration.java | 2 +- .../CalculatedFieldDataValidator.java | 8 +++---- ui-ngx/src/app/core/auth/auth.models.ts | 2 +- ui-ngx/src/app/core/auth/auth.reducer.ts | 2 +- ...ntity-aggregation-component.component.html | 4 ++-- .../entity-aggregation-component.component.ts | 6 ++--- ...enant-profile-configuration.component.html | 6 ++--- ...-tenant-profile-configuration.component.ts | 2 +- ui-ngx/src/app/shared/models/tenant.model.ts | 4 ++-- 36 files changed, 136 insertions(+), 82 deletions(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index d01fa37312..8252a15282 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -47,9 +47,9 @@ SET profile_data = jsonb_set( THEN NULL ELSE to_jsonb(60) END, - 'minAggregationIntervalInSecForCF', + 'minAllowedAggregationIntervalInSecForCF', CASE - WHEN (profile_data -> 'configuration') ? 'minAggregationIntervalInSecForCF' + WHEN (profile_data -> 'configuration') ? 'minAllowedAggregationIntervalInSecForCF' THEN NULL ELSE to_jsonb(60) END @@ -66,7 +66,7 @@ WHERE NOT ( AND (profile_data -> 'configuration') ? 'minAllowedDeduplicationIntervalInSecForCF' AND - (profile_data -> 'configuration') ? 'minAggregationIntervalInSecForCF' + (profile_data -> 'configuration') ? 'minAllowedAggregationIntervalInSecForCF' ); -- UPDATE TENANT PROFILE CONFIGURATION END diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index c8b4c37ead..16081a6d31 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -97,8 +97,8 @@ import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.queue.QueueStatsService; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.dao.resource.TbResourceDataCache; import org.thingsboard.server.dao.resource.ResourceService; +import org.thingsboard.server.dao.resource.TbResourceDataCache; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.rule.RuleNodeStateService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; @@ -664,10 +664,14 @@ public class ActorSystemContext { @Getter private long cfCalculationResultTimeout; - @Value("${actors.calculated_fields.check_interval:120}") + @Value("${actors.calculated_fields.check_interval:60}") @Getter private long cfCheckInterval; + @Value("${actors.alarms.reevaluation_interval:120}") + @Getter + private long alarmRulesReevaluationInterval; + @Autowired @Getter private MqttClientSettings mqttClientSettings; @@ -851,7 +855,7 @@ public class ActorSystemContext { if (arguments != null) { eventBuilder.arguments(JacksonUtil.toString( arguments.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toTbelCfArg())) + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().jsonValue())) )); } if (result != null) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 89f5578760..5d9aeb17ff 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -53,7 +53,6 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; -import org.thingsboard.server.service.cf.ctx.state.aggregation.single.EntityAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -124,12 +123,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (state != null) { state.setCtx(msg.getCtx(), actorCtx); state.setPartition(msg.getPartition()); - if (state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { - relatedEntitiesAggState.scheduleReevaluation(); - } - if (state instanceof EntityAggregationCalculatedFieldState entityAggState) { - entityAggState.fillMissingIntervals(); - } + state.init(true); states.put(cfId, state); } else { removeState(cfId); @@ -140,7 +134,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM log.debug("Processing CF state partition restore msg: {}", msg); for (CalculatedFieldState state : states.values()) { if (msg.getPartition().equals(state.getPartition())) { - state.init(); + state.init(false); } } } @@ -455,7 +449,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private void initState(CalculatedFieldState state, CalculatedFieldCtx ctx) { state.setCtx(ctx, actorCtx); - state.init(); + state.init(false); if (ctx.getCfType() == CalculatedFieldType.GEOFENCING && ctx.isRelationQueryDynamicArguments()) { GeofencingCalculatedFieldState geofencingState = (GeofencingCalculatedFieldState) state; @@ -504,7 +498,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else { if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { String errorMsg = ctx.isInitialized() ? state.getReadinessStatus().errorMsg() : "Calculated field state is not initialized!"; - systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, null, errorMsg); + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, null, errorMsg); } callback.onSuccess(); } diff --git a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java index 7b6d7b5768..9c04cb92bd 100644 --- a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java +++ b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java @@ -165,7 +165,7 @@ public class SystemInfoController extends BaseController { systemParams.setMinAllowedScheduledUpdateIntervalInSecForCF(tenantProfileConfiguration.getMinAllowedScheduledUpdateIntervalInSecForCF()); systemParams.setMaxRelationLevelPerCfArgument(tenantProfileConfiguration.getMaxRelationLevelPerCfArgument()); systemParams.setMinAllowedDeduplicationIntervalInSecForCF(tenantProfileConfiguration.getMinAllowedDeduplicationIntervalInSecForCF()); - systemParams.setMinAggregationIntervalInSecForCF(tenantProfileConfiguration.getMinAggregationIntervalInSecForCF()); + systemParams.setMinAllowedAggregationIntervalInSecForCF(tenantProfileConfiguration.getMinAllowedAggregationIntervalInSecForCF()); systemParams.setTrendzSettings(trendzSettingsService.findTrendzSettings(currentUser.getTenantId())); } systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID)) diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java b/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java index 1a3bdce1f6..6c0f70e17c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java @@ -164,9 +164,14 @@ public class TenantProfileController extends BaseController { " \"warnThreshold\": 0,\n" + " \"maxCalculatedFieldsPerEntity\": 5,\n" + " \"maxArgumentsPerCF\": 10,\n" + + " \"minAllowedScheduledUpdateIntervalInSecForCF\": 60,\n" + + " \"maxRelationLevelPerCfArgument\": 10,\n" + + " \"maxRelatedEntitiesToReturnPerCfArgument\": 100,\n" + " \"maxDataPointsPerRollingArg\": 1000,\n" + " \"maxStateSizeInKBytes\": 32,\n" + " \"maxSingleValueArgumentSizeInKBytes\": 2" + + " \"minAllowedDeduplicationIntervalInSecForCF\": 60" + + " \"minAllowedAggregationIntervalInSecForCF\": 60" + " }\n" + " },\n" + " \"default\": false\n" + diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 6914f6d10a..c22623e109 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -199,12 +199,13 @@ public abstract class AbstractCalculatedFieldProcessingService { } protected Map> fetchEntityAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { - EntityAggregationCalculatedFieldConfiguration aggConfig = (EntityAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - - return aggConfig.getArguments().entrySet().stream() + if (!(ctx.getCalculatedField().getConfiguration() instanceof EntityAggregationCalculatedFieldConfiguration config)) { + return Collections.emptyMap(); + } + return config.getArguments().entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, - entry -> fetchTimeSeries(ctx.getTenantId(), entityId, entry.getValue(), aggConfig.getInterval(), ts) + entry -> fetchTimeSeries(ctx.getTenantId(), entityId, entry.getValue(), config.getInterval(), ts) )); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index 5842363229..d50a125451 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -25,6 +26,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.List; import java.util.Set; import java.util.function.Predicate; +import java.util.stream.Stream; public interface CalculatedFieldCache { @@ -38,9 +40,7 @@ public interface CalculatedFieldCache { List getCalculatedFieldCtxsByEntityId(EntityId entityId); - List getRelatedEntitiesAggCalculatedFieldCtxsByFilter(Predicate relatedEntityFilter); - - List getEntityAggCalculatedFieldCtxsByFilter(Predicate entityAggCfFilter); + Stream getCalculatedFieldCtxsByType(CalculatedFieldType cfType); boolean hasCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 4f2b7a459c..9e51997a20 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -49,6 +49,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Predicate; +import java.util.stream.Stream; @Service @Slf4j @@ -148,21 +149,10 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } @Override - public List getRelatedEntitiesAggCalculatedFieldCtxsByFilter(Predicate relatedEntityFilter) { + public Stream getCalculatedFieldCtxsByType(CalculatedFieldType cfType) { return calculatedFields.values().stream() - .filter(cf -> CalculatedFieldType.RELATED_ENTITIES_AGGREGATION.equals(cf.getType())) - .map(cf -> getCalculatedFieldCtx(cf.getId())) - .filter(relatedEntityFilter) - .toList(); - } - - @Override - public List getEntityAggCalculatedFieldCtxsByFilter(Predicate entityAggCfFilter) { - return calculatedFields.values().stream() - .filter(cf -> CalculatedFieldType.ENTITY_AGGREGATION.equals(cf.getType())) - .map(cf -> getCalculatedFieldCtx(cf.getId())) - .filter(entityAggCfFilter) - .toList(); + .filter(cf -> cfType.equals(cf.getType())) + .map(cf -> getCalculatedFieldCtx(cf.getId())); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 1bc1222d6b..36a44b0d74 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -27,6 +27,7 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -188,13 +189,15 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } } - List entityAggCfCtxs = calculatedFieldCache.getEntityAggCalculatedFieldCtxsByFilter(filter); - if (!entityAggCfCtxs.isEmpty()) { + boolean hasMatchesEntityAggCfs = calculatedFieldCache.getCalculatedFieldCtxsByType(CalculatedFieldType.ENTITY_AGGREGATION).anyMatch(filter); + if (hasMatchesEntityAggCfs) { return true; } - List relatedEntityAggCfCtxs = calculatedFieldCache.getRelatedEntitiesAggCalculatedFieldCtxsByFilter(relatedEntityFilter); - for (CalculatedFieldCtx cfCtx : relatedEntityAggCfCtxs) { + List relatedEntitiesAggregationCfs = calculatedFieldCache.getCalculatedFieldCtxsByType(CalculatedFieldType.RELATED_ENTITIES_AGGREGATION) + .filter(relatedEntityFilter) + .toList(); + for (CalculatedFieldCtx cfCtx : relatedEntitiesAggregationCfs) { if (cfCtx.getCalculatedField().getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig) { RelationPathLevel relation = aggConfig.getRelation(); EntitySearchDirection inverseDirection = switch (relation.direction()) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 55a61d1918..dc23ffa979 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -18,6 +18,8 @@ package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.JsonNode; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; @@ -54,6 +56,10 @@ public interface ArgumentEntry { boolean isEmpty(); + default JsonNode jsonValue() { + return JacksonUtil.valueToTree(toTbelCfArg()); + } + TbelCfArg toTbelCfArg(); boolean isForceResetPrevious(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 20f944e433..e438274ab6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -62,7 +62,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, } @Override - public void init() { + public void init(boolean restored) { } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 5c50f7e95e..518b65dc5f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -96,9 +96,11 @@ public class CalculatedFieldCtx implements Closeable { private Output output; private String expression; private boolean useLatestTs; - private boolean requiresScheduledReevaluation; - private long aggCheckInterval; + private long cfCheckInterval; + private long alarmReevaluationInterval; + + private long lastReevaluationTs; private ActorSystemContext systemContext; private TbelInvokeService tbelInvokeService; @@ -200,7 +202,8 @@ public class CalculatedFieldCtx implements Closeable { if (calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig) { this.useLatestTs = aggConfig.isUseLatestTs(); } - this.aggCheckInterval = systemContext.getCfCheckInterval(); + this.cfCheckInterval = systemContext.getCfCheckInterval(); + this.alarmReevaluationInterval = systemContext.getAlarmRulesReevaluationInterval(); this.systemContext = systemContext; this.tbelInvokeService = systemContext.getTbelInvokeService(); this.relationService = systemContext.getRelationService(); @@ -213,19 +216,28 @@ public class CalculatedFieldCtx implements Closeable { } public boolean isRequiresScheduledReevaluation() { + long now = System.currentTimeMillis(); + long cfCheckIntervalMillis = TimeUnit.SECONDS.toMillis(systemContext.getCfCheckInterval()); if (calculatedField.getConfiguration() instanceof EntityAggregationCalculatedFieldConfiguration entityAggregationConfig) { - long now = System.currentTimeMillis(); Watermark watermark = entityAggregationConfig.getWatermark(); if (watermark != null && watermark.getDuration() > 0) { return true; } - long cfCheckIntervalMillis = TimeUnit.SECONDS.toMillis(systemContext.getCfCheckInterval()); long intervalEndTs = entityAggregationConfig.getInterval().getCurrentIntervalEndTs(); if (now + cfCheckIntervalMillis >= intervalEndTs) { return true; } } - return calculatedField.getConfiguration().requiresScheduledReevaluation(); + long reevaluationIntervalMillis = TimeUnit.SECONDS.toMillis(systemContext.getAlarmRulesReevaluationInterval()); + boolean requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); + if (requiresScheduledReevaluation) { + if (now + cfCheckIntervalMillis >= lastReevaluationTs + reevaluationIntervalMillis) { + lastReevaluationTs = now; + return true; + } + return false; + } + return false; } public void init() { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 64b67d7ab5..e3914cc125 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -27,8 +27,8 @@ import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.state.aggregation.single.EntityAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.EntityAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -63,7 +63,7 @@ public interface CalculatedFieldState extends Closeable { void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx); - void init(); + void init(boolean restored); Map update(Map arguments, CalculatedFieldCtx ctx); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java index ff50cd99b6..8159b1db67 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java @@ -74,6 +74,14 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat deduplicationIntervalMs = SECONDS.toMillis(configuration.getDeduplicationIntervalInSec()); } + @Override + public void init(boolean restored) { + super.init(restored); + if (restored) { + scheduleReevaluation(); + } + } + @Override public void close() { super.close(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntry.java index fac4e403a8..acbaffd5d9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntry.java @@ -26,7 +26,7 @@ public class AggIntervalEntry { private Long endTs; public boolean belongsToInterval(long ts) { - return ts >= startTs && ts <= endTs; + return ts >= startTs && ts < endTs; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationArgumentEntry.java index 9ce47583e3..d738c7d10b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationArgumentEntry.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation.single; +import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; @@ -66,6 +68,11 @@ public class EntityAggregationArgumentEntry implements ArgumentEntry { return aggIntervals.isEmpty(); } + @Override + public JsonNode jsonValue() { + return JacksonUtil.valueToTree(aggIntervals); + } + @Override public TbelCfArg toTbelCfArg() { return null; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java index af7912da46..54f094c289 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java @@ -68,11 +68,19 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt intervalDuration = configuration.getInterval().getIntervalDurationMillis(); Watermark watermark = configuration.getWatermark(); watermarkDuration = watermark == null ? 0 : TimeUnit.SECONDS.toMillis(watermark.getDuration()); - checkInterval = TimeUnit.SECONDS.toMillis(ctx.getAggCheckInterval()); + checkInterval = TimeUnit.SECONDS.toMillis(ctx.getCfCheckInterval()); interval = configuration.getInterval(); metrics = configuration.getMetrics(); } + @Override + public void init(boolean restored) { + super.init(restored); + if (restored) { + fillMissingIntervals(); + } + } + @Override public CalculatedFieldType getType() { return CalculatedFieldType.ENTITY_AGGREGATION; @@ -114,9 +122,7 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt AggIntervalEntry currentInterval = new AggIntervalEntry(interval.getCurrentIntervalStartTs(), interval.getCurrentIntervalEndTs()); arguments.forEach((argName, argumentEntry) -> { var entityAggEntry = (EntityAggregationArgumentEntry) argumentEntry; - if (!entityAggEntry.getAggIntervals().containsKey(currentInterval)) { - entityAggEntry.getAggIntervals().computeIfAbsent(currentInterval, current -> new AggIntervalEntryStatus()); - } + entityAggEntry.getAggIntervals().computeIfAbsent(currentInterval, current -> new AggIntervalEntryStatus()); }); } @@ -244,7 +250,7 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt } ObjectNode resultNode = JacksonUtil.newObjectNode(); if (!metricsNode.isEmpty()) { - resultNode.put("ts", interval.getEndTs()); + resultNode.put("ts", interval.getEndTs() - 1); resultNode.set("values", metricsNode); } result.add(resultNode); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java index b96fde9898..83e08ea67f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java @@ -122,8 +122,11 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { } @Override - public void init() { - super.init(); + public void init(boolean restored) { + super.init(restored); + if (restored) { + return; + } AtomicBoolean reevalNeeded = new AtomicBoolean(false); Map createRules = configuration.getCreateRules(); for (AlarmSeverity severity : AlarmSeverity.values()) { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 1c32d62f6c..cdeb55b7b7 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -531,6 +531,9 @@ actors: calculation_timeout: "${ACTORS_CALCULATION_TIMEOUT_SEC:5}" # Interval in seconds to re-evaluate calculated fields that have a time schedule. 1 minute by default. check_interval: "${ACTORS_CALCULATED_FIELDS_CHECK_INTERVAL_SEC:60}" + alarms: + # Interval in seconds to re-evaluate Alarm rules that have a time schedule. 2 minutes by default. + reevaluation_interval: "${ACTORS_ALARMS_REEVALUATION_INTERVAL_SEC:120}" debug: settings: diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java index d144fe4dcc..cc2d9c8437 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/GeofencingCalculatedFieldStateTest.java @@ -105,7 +105,7 @@ public class GeofencingCalculatedFieldStateTest { ctx.init(); state = new GeofencingCalculatedFieldState(ctx.getEntityId()); state.setCtx(ctx, null); - state.init(); + state.init(false); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java index ddb9f378b0..88cc6972b8 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java @@ -105,7 +105,7 @@ public class PropagationCalculatedFieldStateTest { state = new PropagationCalculatedFieldState(ctx.getEntityId()); state.setCtx(ctx, null); - state.init(); + state.init(false); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index 51e633a232..9691f4a02d 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -88,7 +88,7 @@ public class ScriptCalculatedFieldStateTest { ctx.init(); state = new ScriptCalculatedFieldState(ctx.getEntityId()); state.setCtx(ctx, null); - state.init(); + state.init(false); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index a64f1e4c60..6b25643cdf 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -80,7 +80,7 @@ public class SimpleCalculatedFieldStateTest { ctx.init(); state = new SimpleCalculatedFieldState(ctx.getEntityId()); state.setCtx(ctx, null); - state.init(); + state.init(false); } @Test diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java index c46b466e90..0fa9b2dd78 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java @@ -41,6 +41,6 @@ public class SystemParams { int minAllowedScheduledUpdateIntervalInSecForCF; int maxRelationLevelPerCfArgument; long minAllowedDeduplicationIntervalInSecForCF; - long minAggregationIntervalInSecForCF; + long minAllowedAggregationIntervalInSecForCF; TrendzSettings trendzSettings; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java index 78e74ee64d..eb97538701 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java @@ -17,6 +17,7 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single; import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; import lombok.Data; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; @@ -36,8 +37,13 @@ public class EntityAggregationCalculatedFieldConfiguration implements ArgumentsB @Valid @NotEmpty private Map metrics; + @Valid + @NotNull private AggInterval interval; + @Valid private Watermark watermark; + @Valid + @NotNull private Output output; @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java index 71ccab5f95..f7cae9cd9e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java @@ -17,6 +17,9 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.i import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import lombok.Data; import java.time.DayOfWeek; @@ -32,6 +35,7 @@ import java.time.temporal.TemporalAdjusters; @JsonInclude(JsonInclude.Include.NON_NULL) public abstract class BaseAggInterval implements AggInterval { + @NotBlank protected String tz; protected Long offsetSec; // delay seconds since start of interval diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/Watermark.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/Watermark.java index 3b11681982..b07e6a3012 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/Watermark.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/Watermark.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; +import jakarta.validation.constraints.Min; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -24,6 +25,7 @@ import lombok.NoArgsConstructor; @NoArgsConstructor public class Watermark { + @Min(0) private long duration; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index 62af00300e..7587f563ab 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -189,7 +189,7 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura @Schema(example = "60") private long minAllowedDeduplicationIntervalInSecForCF = 60; @Schema(example = "60") - private long minAggregationIntervalInSecForCF = 60; + private long minAllowedAggregationIntervalInSecForCF = 60; @Override public long getProfileThreshold(ApiUsageRecordKey key) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index 17d980488d..fa688a0a9e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -123,10 +123,10 @@ public class CalculatedFieldDataValidator extends DataValidator if (!(calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfiguration)) { return; } - long minAllowedDeduplicationInterval = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMinAllowedDeduplicationIntervalInSecForCF); - if (aggConfiguration.getDeduplicationIntervalInSec() < minAllowedDeduplicationInterval) { + long minDeduplicationInterval = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMinAllowedDeduplicationIntervalInSecForCF); + if (aggConfiguration.getDeduplicationIntervalInSec() < minDeduplicationInterval) { throw new IllegalArgumentException("Deduplication interval is less than configured " + - "minimum allowed interval in tenant profile: " + minAllowedDeduplicationInterval); + "minimum allowed interval in tenant profile: " + minDeduplicationInterval); } } @@ -134,7 +134,7 @@ public class CalculatedFieldDataValidator extends DataValidator if (!(calculatedField.getConfiguration() instanceof EntityAggregationCalculatedFieldConfiguration aggConfiguration)) { return; } - long minAggregationIntervalInSec = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMinAggregationIntervalInSecForCF); + long minAggregationIntervalInSec = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMinAllowedAggregationIntervalInSecForCF); if (minAggregationIntervalInSec <= 0) { return; } diff --git a/ui-ngx/src/app/core/auth/auth.models.ts b/ui-ngx/src/app/core/auth/auth.models.ts index 582a407841..21759fbca0 100644 --- a/ui-ngx/src/app/core/auth/auth.models.ts +++ b/ui-ngx/src/app/core/auth/auth.models.ts @@ -32,7 +32,7 @@ export interface SysParamsState { maxDataPointsPerRollingArg: number; maxArgumentsPerCF: number; minAllowedDeduplicationIntervalInSecForCF: number; - minAggregationIntervalInSecForCF: number; + minAllowedAggregationIntervalInSecForCF: number; minAllowedScheduledUpdateIntervalInSecForCF: number; maxRelationLevelPerCfArgument: number; ruleChainDebugPerTenantLimitsConfiguration?: string; diff --git a/ui-ngx/src/app/core/auth/auth.reducer.ts b/ui-ngx/src/app/core/auth/auth.reducer.ts index 8f8258d7b9..af040a6d53 100644 --- a/ui-ngx/src/app/core/auth/auth.reducer.ts +++ b/ui-ngx/src/app/core/auth/auth.reducer.ts @@ -34,7 +34,7 @@ const emptyUserAuthState: AuthPayload = { maxResourceSize: 0, maxArgumentsPerCF: 0, minAllowedDeduplicationIntervalInSecForCF: 0, - minAggregationIntervalInSecForCF: 0, + minAllowedAggregationIntervalInSecForCF: 0, minAllowedScheduledUpdateIntervalInSecForCF: 0, maxRelationLevelPerCfArgument: 0, maxDataPointsPerRollingArg: 0, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.html index 2907472143..e4a40c3cdd 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.html @@ -62,14 +62,14 @@ @if (entityAggregationConfiguration.get('interval.type').value === AggIntervalType.CUSTOM) { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts index 8da5c47606..ae63d0f7a5 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts @@ -85,7 +85,7 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor @Input({required: true}) entityName: string; - readonly minAggregationIntervalInSecForCF = getCurrentAuthState(this.store).minAggregationIntervalInSecForCF; + readonly minAllowedAggregationIntervalInSecForCF = getCurrentAuthState(this.store).minAllowedAggregationIntervalInSecForCF; readonly DayInSec = DAY / SECOND; entityAggregationConfiguration = this.fb.group({ @@ -94,9 +94,9 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor interval: this.fb.group({ type: [AggIntervalType.HOUR], tz: ['', Validators.required], - durationSec: [this.minAggregationIntervalInSecForCF, Validators.required], + durationSec: [this.minAllowedAggregationIntervalInSecForCF, Validators.required], allowOffsetSec: [false], - offsetSec: [this.minAggregationIntervalInSecForCF > 60 ? MINUTE / SECOND : 1, Validators.required], + offsetSec: [this.minAllowedAggregationIntervalInSecForCF > 60 ? MINUTE / SECOND : 1, Validators.required], }), allowWatermark: [false], watermark: this.fb.group({ diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html index 2c2f7b5973..4c311333ec 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -344,12 +344,12 @@ tenant-profile.min-allowed-aggregation-interval - + {{ 'tenant-profile.min-allowed-aggregation-interval-required' | translate}} - + {{ 'tenant-profile.min-allowed-aggregation-interval-range' | translate}} diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts index 16997d18a7..a61a1aa1f8 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts @@ -117,7 +117,7 @@ export class DefaultTenantProfileConfigurationComponent implements ControlValueA maxArgumentsPerCF: [0, [Validators.required, Validators.min(0)]], maxRelationLevelPerCfArgument: [1, [Validators.required, Validators.min(1)]], minAllowedDeduplicationIntervalInSecForCF: [0, [Validators.required, Validators.min(0)]], - minAggregationIntervalInSecForCF: [0, [Validators.required, Validators.min(0)]], + minAllowedAggregationIntervalInSecForCF: [0, [Validators.required, Validators.min(0)]], maxRelatedEntitiesToReturnPerCfArgument: [1, [Validators.required, Validators.min(1)]], minAllowedScheduledUpdateIntervalInSecForCF: [0, [Validators.required, Validators.min(0)]], maxDataPointsPerRollingArg: [0, [Validators.required, Validators.min(0)]], diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 7a96051239..0cfa8df888 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -108,7 +108,7 @@ export interface DefaultTenantProfileConfiguration { maxArgumentsPerCF: number; maxRelationLevelPerCfArgument: number; minAllowedDeduplicationIntervalInSecForCF: number; - minAggregationIntervalInSecForCF: number; + minAllowedAggregationIntervalInSecForCF: number; maxRelatedEntitiesToReturnPerCfArgument: number; minAllowedScheduledUpdateIntervalInSecForCF: number; maxDataPointsPerRollingArg: number; @@ -177,7 +177,7 @@ export function createTenantProfileConfiguration(type: TenantProfileType): Tenan maxDataPointsPerRollingArg: 1000, maxRelationLevelPerCfArgument: 10, minAllowedDeduplicationIntervalInSecForCF: 60, - minAggregationIntervalInSecForCF: 60, + minAllowedAggregationIntervalInSecForCF: 60, maxRelatedEntitiesToReturnPerCfArgument: 100, minAllowedScheduledUpdateIntervalInSecForCF: 0, maxStateSizeInKBytes: 32, From a71af9b1bfd5545ba6f21b84c114bf4426af37d8 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Tue, 11 Nov 2025 16:21:16 +0200 Subject: [PATCH 30/40] lwm2m: LWM2M_DTLS_CONNECTION_ID_LENGTH=8 --- application/src/main/resources/thingsboard.yml | 4 ++-- .../org/thingsboard/server/coapserver/TbCoapDtlsSettings.java | 2 +- .../transport/lwm2m/config/LwM2MTransportServerConfig.java | 2 +- transport/coap/src/main/resources/tb-coap-transport.yml | 2 +- transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 889df54848..d5b8cfe15e 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1157,7 +1157,7 @@ transport: # - A value of 0 means we accept using CID but will not generate one for foreign peer (enables support but not for incoming traffic). # - A value between 0 and <= 4: SingleNodeConnectionIdGenerator is used # - A value that are > 4: MultiNodeConnectionIdGenerator is used - connection_id_length: "${LWM2M_DTLS_CONNECTION_ID_LENGTH:}" + connection_id_length: "${LWM2M_DTLS_CONNECTION_ID_LENGTH:8}" server: # LwM2M Server ID id: "${LWM2M_SERVER_ID:123}" @@ -1345,7 +1345,7 @@ coap: # - A value of 0 means we accept using CID but will not generate one for foreign peer (enables support but not for incoming traffic). # - A value between 0 and <= 4: SingleNodeConnectionIdGenerator is used # - A value that are > 4: MultiNodeConnectionIdGenerator is used - connection_id_length: "${COAP_DTLS_CONNECTION_ID_LENGTH:}" + connection_id_length: "${COAP_DTLS_CONNECTION_ID_LENGTH:8}" # Specify the MTU (Maximum Transmission Unit). # Should be used if LAN MTU is not used, e.g. if IP tunnels are used or if the client uses a smaller value than the LAN MTU. # Default = 1024 diff --git a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapDtlsSettings.java b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapDtlsSettings.java index 56705b5608..f98dc7d3c6 100644 --- a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapDtlsSettings.java +++ b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapDtlsSettings.java @@ -66,7 +66,7 @@ public class TbCoapDtlsSettings { @Value("${coap.dtls.retransmission_timeout:9000}") private int dtlsRetransmissionTimeout; - @Value("${coap.dtls.connection_id_length:}") + @Value("${coap.dtls.connection_id_length:8}") private Integer cIdLength; @Value("${coap.dtls.max_transmission_unit:1024}") diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/config/LwM2MTransportServerConfig.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/config/LwM2MTransportServerConfig.java index c1af8f553a..bf65b95dd2 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/config/LwM2MTransportServerConfig.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/config/LwM2MTransportServerConfig.java @@ -43,7 +43,7 @@ public class LwM2MTransportServerConfig implements LwM2MSecureServerConfig { private int dtlsRetransmissionTimeout; @Getter - @Value("${transport.lwm2m.dtls.connection_id_length:}") + @Value("${transport.lwm2m.dtls.connection_id_length:8}") private Integer dtlsCidLength; @Getter diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index f40a09c753..2f3942f847 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -193,7 +193,7 @@ coap: # - A value of 0 means we accept using CID but will not generate one for foreign peer (enables support but not for incoming traffic). # - A value between 0 and <= 4: SingleNodeConnectionIdGenerator is used # - A value that are > 4: MultiNodeConnectionIdGenerator is used - connection_id_length: "${COAP_DTLS_CONNECTION_ID_LENGTH:}" + connection_id_length: "${COAP_DTLS_CONNECTION_ID_LENGTH:8}" # Specify the MTU (Maximum Transmission Unit). # Should be used if LAN MTU is not used, e.g. if IP tunnels are used or if the client uses a smaller value than the LAN MTU. # Default = 1024 diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index 0895bfa676..323f80b999 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -173,7 +173,7 @@ transport: # - A value of 0 means we accept using CID but will not generate one for foreign peer (enables support but not for incoming traffic). # - A value between 0 and <= 4: SingleNodeConnectionIdGenerator is used # - A value that are > 4: MultiNodeConnectionIdGenerator is used - connection_id_length: "${LWM2M_DTLS_CONNECTION_ID_LENGTH:}" + connection_id_length: "${LWM2M_DTLS_CONNECTION_ID_LENGTH:8}" server: # LwM2M Server ID id: "${LWM2M_SERVER_ID:123}" From 1de8e8cd35280bc8a6d15c2034b2e63f0af0c0c4 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 12 Nov 2025 17:00:47 +0200 Subject: [PATCH 31/40] fixed intervals --- .../aggregation/single/AggIntervalEntry.java | 4 + ...EntityAggregationCalculatedFieldState.java | 22 +-- .../EntityAggregationCalculatedFieldTest.java | 8 +- .../single/interval/AggInterval.java | 15 +- .../single/interval/BaseAggInterval.java | 114 ++++---------- .../single/interval/CustomInterval.java | 38 ++--- .../single/interval/DayInterval.java | 18 +++ .../single/interval/HourInterval.java | 20 +++ .../single/interval/MonthInterval.java | 18 +++ .../single/interval/QuarterInterval.java | 24 +++ .../single/interval/WeekInterval.java | 21 +++ .../single/interval/WeekSunSatInterval.java | 21 +++ .../single/interval/YearInterval.java | 22 +++ .../single/interval/AggIntervalTest.java | 139 ++++++++++++++++++ .../CalculatedFieldDataValidator.java | 2 +- 15 files changed, 361 insertions(+), 125 deletions(-) create mode 100644 common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntry.java index acbaffd5d9..338e667dd2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/AggIntervalEntry.java @@ -29,4 +29,8 @@ public class AggIntervalEntry { return ts >= startTs && ts < endTs; } + public long getIntervalDuration() { + return endTs - startTs; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java index 54f094c289..a1208c5054 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java @@ -37,6 +37,9 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.BaseCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; @@ -49,7 +52,6 @@ import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDe public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldState { private AggInterval interval; - private long intervalDuration; private long watermarkDuration; private long checkInterval; private Map metrics; @@ -65,7 +67,6 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt super.setCtx(ctx, actorCtx); this.cfProcessingService = ctx.getCfProcessingService(); var configuration = (EntityAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - intervalDuration = configuration.getInterval().getIntervalDurationMillis(); Watermark watermark = configuration.getWatermark(); watermarkDuration = watermark == null ? 0 : TimeUnit.SECONDS.toMillis(watermark.getDuration()); checkInterval = TimeUnit.SECONDS.toMillis(ctx.getCfCheckInterval()); @@ -127,18 +128,21 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt } public void fillMissingIntervals() { + ZoneId zoneId = interval.getZoneId(); long currentIntervalEndTs = interval.getCurrentIntervalEndTs(); - long intervalDuration = interval.getIntervalDurationMillis(); + Map> intervals = getIntervals(); AggIntervalEntry lastIntervalEntry = intervals.keySet().stream().max(Comparator.comparing(AggIntervalEntry::getEndTs)).orElse(null); if (lastIntervalEntry == null) { return; } - long nextStartTs = lastIntervalEntry.getEndTs(); - long nextEndTs = nextStartTs + intervalDuration; + ZonedDateTime nextStart = Instant.ofEpochMilli(lastIntervalEntry.getEndTs()).atZone(zoneId); + ZonedDateTime nextEnd = interval.getNextIntervalStart(nextStart); - while (nextEndTs <= currentIntervalEndTs) { + while (nextEnd.toInstant().toEpochMilli() <= currentIntervalEndTs) { + long nextStartTs = nextStart.toInstant().toEpochMilli(); + long nextEndTs = nextEnd.toInstant().toEpochMilli(); AggIntervalEntry missing = new AggIntervalEntry(nextStartTs, nextEndTs); arguments.forEach((argName, argumentEntry) -> { @@ -147,8 +151,8 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt entityAggEntry.getAggIntervals().computeIfAbsent(missing, missingInterval -> intervalEntryStatus); }); - nextStartTs = nextEndTs; - nextEndTs += intervalDuration; + nextStart = nextEnd; + nextEnd = interval.getNextIntervalStart(nextStart); } } @@ -174,7 +178,7 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt if (now - endTs > watermarkDuration) { handleExpiredInterval(intervalEntry, args, results); expiredIntervals.add(intervalEntry); - } else if (now - startTs >= intervalDuration) { + } else if (now - startTs >= intervalEntry.getIntervalDuration()) { handleActiveInterval(intervalEntry, args, results); } } diff --git a/application/src/test/java/org/thingsboard/server/cf/EntityAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/EntityAggregationCalculatedFieldTest.java index fb92ebe3af..7a8f5bb625 100644 --- a/application/src/test/java/org/thingsboard/server/cf/EntityAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/EntityAggregationCalculatedFieldTest.java @@ -95,7 +95,7 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest public void testCreateCf_checkAggregation() throws Exception { Device device = createDevice("Device", "1234567890111"); - CustomInterval customInterval = new CustomInterval(30L, 0L, "Europe/Kyiv"); + CustomInterval customInterval = new CustomInterval("Europe/Kyiv", 30L, 0L); long currentIntervalStartTs = customInterval.getCurrentIntervalStartTs(); long currentIntervalEndTs = customInterval.getCurrentIntervalEndTs(); @@ -108,7 +108,7 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest postTelemetry(device.getId(), String.format("{\"ts\": \"%s\", \"values\": {\"energy\":180}}", tsInInterval_2)); postTelemetry(device.getId(), String.format("{\"ts\": \"%s\", \"values\": {\"energy\":120}}", tsInInterval_3)); - long interval = customInterval.getIntervalDurationMillis(); + long interval = customInterval.getCurrentIntervalDurationMillis(); Watermark watermark = new Watermark(60); CalculatedField totalConsumptionCF = createTotalConsumptionCF(device.getId(), customInterval, watermark); @@ -126,7 +126,7 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest public void testCreateCf_checkAggregationDuringWatermark() throws Exception { Device device = createDevice("Device", "1234567890111"); - CustomInterval customInterval = new CustomInterval(30L, 0L, "Europe/Kyiv"); + CustomInterval customInterval = new CustomInterval("Europe/Kyiv", 30L, 0L); long currentIntervalStartTs = customInterval.getCurrentIntervalStartTs(); long currentIntervalEndTs = customInterval.getCurrentIntervalEndTs(); @@ -139,7 +139,7 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest postTelemetry(device.getId(), String.format("{\"ts\": \"%s\", \"values\": {\"energy\":180}}", tsInInterval_2)); postTelemetry(device.getId(), String.format("{\"ts\": \"%s\", \"values\": {\"energy\":120}}", tsInInterval_3)); - long interval = customInterval.getIntervalDurationMillis(); + long interval = customInterval.getCurrentIntervalDurationMillis(); Watermark watermark = new Watermark(60); CalculatedField totalConsumptionCF = createTotalConsumptionCF(device.getId(), customInterval, watermark); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java index b534002885..6bff7b4398 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java @@ -20,6 +20,9 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import java.time.ZoneId; +import java.time.ZonedDateTime; + @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, @@ -42,15 +45,21 @@ public interface AggInterval { AggIntervalType getType(); @JsonIgnore - long getIntervalDurationMillis(); + ZoneId getZoneId(); + + @JsonIgnore + long getCurrentIntervalDurationMillis(); @JsonIgnore long getCurrentIntervalStartTs(); + long getDateTimeIntervalStartTs(ZonedDateTime dateTime); + @JsonIgnore long getCurrentIntervalEndTs(); - @JsonIgnore - long getDelayUntilIntervalEnd(); + long getDateTimeIntervalEndTs(ZonedDateTime dateTime); + + ZonedDateTime getNextIntervalStart(ZonedDateTime currentStart); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java index f7cae9cd9e..53f7117bfc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java @@ -15,54 +15,50 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; -import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; +import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; -import java.time.DayOfWeek; -import java.time.Duration; -import java.time.LocalDate; -import java.time.LocalTime; import java.time.ZoneId; import java.time.ZonedDateTime; -import java.time.temporal.ChronoUnit; -import java.time.temporal.TemporalAdjusters; @Data @JsonInclude(JsonInclude.Include.NON_NULL) +@AllArgsConstructor +@NoArgsConstructor public abstract class BaseAggInterval implements AggInterval { @NotBlank protected String tz; protected Long offsetSec; // delay seconds since start of interval - @JsonIgnore - protected long getOffsetSec() { + @Override + public ZoneId getZoneId() { + return ZoneId.of(tz); + } + + protected long getOffset() { return offsetSec != null ? offsetSec : 0L; } @Override - public long getIntervalDurationMillis() { - return switch (getType()) { - case HOUR -> Duration.ofHours(1).toMillis(); - case DAY -> Duration.ofDays(1).toMillis(); - case WEEK, WEEK_SUN_SAT -> Duration.ofDays(7L).toMillis(); - case MONTH -> Duration.ofDays(Math.round(30)).toMillis(); // average - case QUARTER -> Duration.ofDays(Math.round(91)).toMillis(); - case YEAR -> Duration.ofDays(Math.round(365)).toMillis(); - default -> throw new IllegalArgumentException("Unsupported type: " + getType()); - }; + public long getCurrentIntervalDurationMillis() { + return getCurrentIntervalEndTs() - getCurrentIntervalStartTs(); } @Override public long getCurrentIntervalStartTs() { - ZoneId zoneId = ZoneId.of(tz); + ZoneId zoneId = getZoneId(); ZonedDateTime now = ZonedDateTime.now(zoneId); - long offset = getOffsetSec(); - ZonedDateTime shiftedNow = now.minusSeconds(offset); + return getDateTimeIntervalStartTs(now); + } + + @Override + public long getDateTimeIntervalStartTs(ZonedDateTime dateTime) { + long offset = getOffset(); + ZonedDateTime shiftedNow = dateTime.minusSeconds(offset); ZonedDateTime alignedStart = getAlignedBoundary(shiftedNow, false); ZonedDateTime actualStart = alignedStart.plusSeconds(offset); return actualStart.toInstant().toEpochMilli(); @@ -70,72 +66,20 @@ public abstract class BaseAggInterval implements AggInterval { @Override public long getCurrentIntervalEndTs() { - ZoneId zoneId = ZoneId.of(tz); + ZoneId zoneId = getZoneId(); ZonedDateTime now = ZonedDateTime.now(zoneId); - long offset = getOffsetSec(); - ZonedDateTime shiftedNow = now.minusSeconds(offset); - ZonedDateTime alignedEnd = getAlignedBoundary(shiftedNow, true); - ZonedDateTime actualEnd = alignedEnd.plusSeconds(offset); - return actualEnd.toInstant().toEpochMilli(); + return getDateTimeIntervalEndTs(now); } @Override - public long getDelayUntilIntervalEnd() { - long currentIntervalEndTs = getCurrentIntervalEndTs(); - long now = System.currentTimeMillis(); - return currentIntervalEndTs - now; - } - - protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { - return switch (getType()) { - case HOUR -> alignByHours(reference, next); - case DAY -> alignByDays(reference, next); - case WEEK -> alignByWeeks(reference, DayOfWeek.MONDAY, next); - case WEEK_SUN_SAT -> alignByWeeks(reference, DayOfWeek.SUNDAY, next); - case MONTH -> alignByMonths(reference, next); - case QUARTER -> alignByQuarters(reference, next); - case YEAR -> alignByYears(reference, next); - default -> throw new IllegalArgumentException("Unsupported interval type: " + getType()); - }; - } - - private ZonedDateTime alignByHours(ZonedDateTime now, boolean next) { - ZonedDateTime base = now.truncatedTo(ChronoUnit.HOURS); - return next ? base.plusHours(1) : base; - } - - private ZonedDateTime alignByDays(ZonedDateTime now, boolean next) { - ZonedDateTime base = now.truncatedTo(ChronoUnit.DAYS); - return next ? base.plusDays(1) : base; - } - - private ZonedDateTime alignByWeeks(ZonedDateTime now, DayOfWeek startOfWeek, boolean next) { - ZonedDateTime startOfWeekDate = now.with(TemporalAdjusters.previousOrSame(startOfWeek)) - .truncatedTo(ChronoUnit.DAYS); - return next ? startOfWeekDate.plusWeeks(1) : startOfWeekDate; - } - - private ZonedDateTime alignByMonths(ZonedDateTime now, boolean next) { - ZonedDateTime base = now.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS); - return next ? base.plusMonths(1) : base; - } - - private ZonedDateTime alignByQuarters(ZonedDateTime now, boolean next) { - int month = now.getMonthValue(); - int quarterStartMonth = ((month - 1) / 3) * 3 + 1; // 1, 4, 7, 10 - ZonedDateTime base = ZonedDateTime.of( - LocalDate.of(now.getYear(), quarterStartMonth, 1), - LocalTime.MIDNIGHT, - now.getZone()); - return next ? base.plusMonths(3) : base; + public long getDateTimeIntervalEndTs(ZonedDateTime dateTime) { + long offset = getOffset(); + ZonedDateTime shiftedNow = dateTime.minusSeconds(offset); + ZonedDateTime alignedEnd = getAlignedBoundary(shiftedNow, true); + ZonedDateTime actualEnd = alignedEnd.plusSeconds(offset); + return actualEnd.toInstant().toEpochMilli(); } - private ZonedDateTime alignByYears(ZonedDateTime now, boolean next) { - ZonedDateTime base = ZonedDateTime.of( - LocalDate.of(now.getYear(), 1, 1), - LocalTime.MIDNIGHT, - now.getZone()); - return next ? base.plusYears(1) : base; - } + protected abstract ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java index c8e3ee15a6..39b2f59b15 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java @@ -20,9 +20,8 @@ import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import java.time.Duration; -import java.time.ZoneId; +import java.time.Instant; import java.time.ZonedDateTime; -import java.util.concurrent.TimeUnit; @EqualsAndHashCode(callSuper = true) @Data @@ -31,9 +30,8 @@ public class CustomInterval extends BaseAggInterval { private Long durationSec; - public CustomInterval(Long durationSec, Long offsetMillis, String tz) { - this.tz = tz; - this.offsetSec = offsetMillis; + public CustomInterval(String tz, Long offsetSec, Long durationSec) { + super(tz, offsetSec); this.durationSec = durationSec; } @@ -43,32 +41,26 @@ public class CustomInterval extends BaseAggInterval { } @Override - public long getIntervalDurationMillis() { - return Duration.ofSeconds(durationSec).toMillis(); + public long getCurrentIntervalDurationMillis() { + return getDurationMillis(); } - @Override - public long getCurrentIntervalStartTs() { - ZoneId zoneId = ZoneId.of(tz); - ZonedDateTime now = ZonedDateTime.now(zoneId); - ZonedDateTime shiftedNow = now.minusSeconds(getOffsetSec()); - - long durationMillis = getIntervalDurationMillis(); - long shiftedNowMillis = shiftedNow.toInstant().toEpochMilli(); - long alignedStartMillis = (shiftedNowMillis / durationMillis) * durationMillis; - - long offsetMillis = TimeUnit.SECONDS.toMillis(getOffsetSec()); - return alignedStartMillis + offsetMillis; + private long getDurationMillis() { + return Duration.ofSeconds(durationSec).toMillis(); } @Override - public long getCurrentIntervalEndTs() { - return getCurrentIntervalStartTs() + getIntervalDurationMillis(); + protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { + long durationMillis = getDurationMillis(); + long nowMillis = reference.toInstant().toEpochMilli(); + long alignedStartMillis = (nowMillis / durationMillis) * durationMillis; + ZonedDateTime aligned = Instant.ofEpochMilli(alignedStartMillis).atZone(getZoneId()); + return next ? aligned.plusSeconds(durationSec) : aligned; } @Override - public long getDelayUntilIntervalEnd() { - return getCurrentIntervalEndTs() - System.currentTimeMillis(); + public ZonedDateTime getNextIntervalStart(ZonedDateTime currentStart) { + return currentStart.plusSeconds(durationSec); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java index e5f48d3116..37e75c9ee6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java @@ -18,6 +18,9 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.i import lombok.Data; import lombok.NoArgsConstructor; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; + @Data @NoArgsConstructor public class DayInterval extends BaseAggInterval { @@ -27,4 +30,19 @@ public class DayInterval extends BaseAggInterval { return AggIntervalType.DAY; } + public DayInterval(String tz, Long offsetSec) { + super(tz, offsetSec); + } + + @Override + protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { + ZonedDateTime base = reference.truncatedTo(ChronoUnit.DAYS); + return next ? base.plusDays(1) : base; + } + + @Override + public ZonedDateTime getNextIntervalStart(ZonedDateTime currentStart) { + return currentStart.plusDays(1); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java index dfd7b7efda..1cac0017e7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java @@ -16,15 +16,35 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; + +@EqualsAndHashCode(callSuper = true) @Data @NoArgsConstructor public class HourInterval extends BaseAggInterval { + public HourInterval(String tz, Long offsetSec) { + super(tz, offsetSec); + } + @Override public AggIntervalType getType() { return AggIntervalType.HOUR; } + @Override + protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { + ZonedDateTime base = reference.truncatedTo(ChronoUnit.HOURS); + return next ? base.plusHours(1) : base; + } + + @Override + public ZonedDateTime getNextIntervalStart(ZonedDateTime currentStart) { + return currentStart.plusHours(1); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java index 91fc3d0413..0a540e49cd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java @@ -18,6 +18,9 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.i import lombok.Data; import lombok.NoArgsConstructor; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; + @Data @NoArgsConstructor public class MonthInterval extends BaseAggInterval { @@ -27,4 +30,19 @@ public class MonthInterval extends BaseAggInterval { return AggIntervalType.MONTH; } + public MonthInterval(String tz, Long offsetSec) { + super(tz, offsetSec); + } + + @Override + protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { + ZonedDateTime base = reference.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS); + return next ? base.plusMonths(1) : base; + } + + @Override + public ZonedDateTime getNextIntervalStart(ZonedDateTime currentStart) { + return currentStart.plusMonths(1); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/QuarterInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/QuarterInterval.java index eb774b2341..bd27c681f8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/QuarterInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/QuarterInterval.java @@ -18,6 +18,10 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.i import lombok.Data; import lombok.NoArgsConstructor; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZonedDateTime; + @Data @NoArgsConstructor public class QuarterInterval extends BaseAggInterval { @@ -27,4 +31,24 @@ public class QuarterInterval extends BaseAggInterval { return AggIntervalType.QUARTER; } + public QuarterInterval(String tz, Long offsetSec) { + super(tz, offsetSec); + } + + @Override + protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { + int month = reference.getMonthValue(); + int quarterStartMonth = ((month - 1) / 3) * 3 + 1; // 1, 4, 7, 10 + ZonedDateTime base = ZonedDateTime.of( + LocalDate.of(reference.getYear(), quarterStartMonth, 1), + LocalTime.MIDNIGHT, + reference.getZone()); + return next ? base.plusMonths(3) : base; + } + + @Override + public ZonedDateTime getNextIntervalStart(ZonedDateTime currentStart) { + return currentStart.plusMonths(3); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java index 2ee5d5f81c..381fb3bb66 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java @@ -18,6 +18,11 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.i import lombok.Data; import lombok.NoArgsConstructor; +import java.time.DayOfWeek; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAdjusters; + @Data @NoArgsConstructor public class WeekInterval extends BaseAggInterval { @@ -27,4 +32,20 @@ public class WeekInterval extends BaseAggInterval { return AggIntervalType.WEEK; } + public WeekInterval(String tz, Long offsetSec) { + super(tz, offsetSec); + } + + @Override + protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { + ZonedDateTime startOfWeekDate = reference.with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)) + .truncatedTo(ChronoUnit.DAYS); + return next ? startOfWeekDate.plusWeeks(1) : startOfWeekDate; + } + + @Override + public ZonedDateTime getNextIntervalStart(ZonedDateTime currentStart) { + return currentStart.plusWeeks(1); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java index f2d403c173..242f3b7914 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java @@ -18,6 +18,11 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.i import lombok.Data; import lombok.NoArgsConstructor; +import java.time.DayOfWeek; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAdjusters; + @Data @NoArgsConstructor public class WeekSunSatInterval extends BaseAggInterval { @@ -27,4 +32,20 @@ public class WeekSunSatInterval extends BaseAggInterval { return AggIntervalType.WEEK_SUN_SAT; } + public WeekSunSatInterval(String tz, Long offsetSec) { + super(tz, offsetSec); + } + + @Override + protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { + ZonedDateTime startOfWeekDate = reference.with(TemporalAdjusters.previousOrSame(DayOfWeek.SUNDAY)) + .truncatedTo(ChronoUnit.DAYS); + return next ? startOfWeekDate.plusWeeks(1) : startOfWeekDate; + } + + @Override + public ZonedDateTime getNextIntervalStart(ZonedDateTime currentStart) { + return currentStart.plusWeeks(1); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java index 23aedf4932..83c8f58301 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java @@ -18,6 +18,10 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.i import lombok.Data; import lombok.NoArgsConstructor; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZonedDateTime; + @Data @NoArgsConstructor public class YearInterval extends BaseAggInterval { @@ -27,4 +31,22 @@ public class YearInterval extends BaseAggInterval { return AggIntervalType.YEAR; } + public YearInterval(String tz, Long offsetSec) { + super(tz, offsetSec); + } + + @Override + protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { + ZonedDateTime base = ZonedDateTime.of( + LocalDate.of(reference.getYear(), 1, 1), + LocalTime.MIDNIGHT, + reference.getZone()); + return next ? base.plusYears(1) : base; + } + + @Override + public ZonedDateTime getNextIntervalStart(ZonedDateTime currentStart) { + return currentStart.plusYears(1); + } + } diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalTest.java new file mode 100644 index 0000000000..38fda87879 --- /dev/null +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalTest.java @@ -0,0 +1,139 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.function.LongFunction; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +public class AggIntervalTest { + + private static final String TZ = "Europe/Kiev"; + + @ParameterizedTest + @MethodSource("intervals") + void testGetStartAndEndWithoutOffset(LongFunction intervalCreator) { + AggInterval interval = intervalCreator.apply(0L); + + ZonedDateTime dateTime = ZonedDateTime.of( + // 2025.11.11 00:00:00 + 2025, 11, 11, 0, 0, 0, 0, ZoneId.of(TZ) + ); + long startTs = interval.getDateTimeIntervalStartTs(dateTime); + long endTs = interval.getDateTimeIntervalEndTs(dateTime); + + assertThat(endTs).isGreaterThan(startTs); + assertThat(endTs - startTs).isEqualTo(interval.getCurrentIntervalDurationMillis()); + } + + @ParameterizedTest + @MethodSource("intervals") + void testApplyOffset(LongFunction intervalCreator) { + long offsetSec = TimeUnit.MINUTES.toSeconds(15); + AggInterval intervalWithOffset = intervalCreator.apply(offsetSec); + AggInterval intervalNoOffset = intervalCreator.apply(0L); + + ZonedDateTime dateTime = ZonedDateTime.of( + // 2025.11.11 11:20:00 - chosen so 15m offset shifts into a new interval + 2025, 11, 11, 11, 20, 0, 0, ZoneId.of(TZ) + ); + + long startWithOffsetTs = intervalWithOffset.getDateTimeIntervalStartTs(dateTime); + long startNoOffsetTs = intervalNoOffset.getDateTimeIntervalStartTs(dateTime); + + ZonedDateTime startWithOffset = Instant.ofEpochMilli(startWithOffsetTs).atZone(intervalWithOffset.getZoneId()); + ZonedDateTime startNoOffset = Instant.ofEpochMilli(startNoOffsetTs).atZone(intervalNoOffset.getZoneId()); + + long actualOffset = Duration.between(startNoOffset, startWithOffset).toSeconds(); + assertThat(actualOffset).isEqualTo(offsetSec); + } + + private static Stream intervals() { + return Stream.of( + Arguments.of((LongFunction) offset -> new HourInterval(TZ, offset)), + Arguments.of((LongFunction) offset -> new DayInterval(TZ, offset)), + Arguments.of((LongFunction) offset -> new WeekInterval(TZ, offset)), + Arguments.of((LongFunction) offset -> new WeekSunSatInterval(TZ, offset)), + Arguments.of((LongFunction) offset -> new MonthInterval(TZ, offset)), + Arguments.of((LongFunction) offset -> new QuarterInterval(TZ, offset)), + Arguments.of((LongFunction) offset -> new YearInterval(TZ, offset)), + Arguments.of((LongFunction) offset -> new CustomInterval(TZ, offset, TimeUnit.HOURS.toSeconds(4))) + ); + } + + @ParameterizedTest + @MethodSource("nextIntervalFromExactDate") + void testNextIntervalFromExactDate(LongFunction intervalCreator, Function expectedDateTimeFunction) { + AggInterval interval = intervalCreator.apply(0L); + + ZonedDateTime currentStart = ZonedDateTime.of( + 2025, 11, 11, 0, 0, 0, 0, ZoneId.of(TZ) + ); + + ZonedDateTime nextStart = interval.getNextIntervalStart(currentStart); + + assertThat(nextStart).isEqualTo(expectedDateTimeFunction.apply(currentStart)); + } + + private static Stream nextIntervalFromExactDate() { + return Stream.of( + Arguments.of( + (LongFunction) offset -> new HourInterval(TZ, offset), + (Function) currentInterval -> currentInterval.plusHours(1) + ), + Arguments.of( + (LongFunction) offset -> new DayInterval(TZ, offset), + (Function) currentInterval -> currentInterval.plusDays(1) + ), + Arguments.of( + (LongFunction) offset -> new WeekInterval(TZ, offset), + (Function) currentInterval -> currentInterval.plusWeeks(1) + ), + Arguments.of( + (LongFunction) offset -> new WeekSunSatInterval(TZ, offset), + (Function) currentInterval -> currentInterval.plusWeeks(1) + ), + Arguments.of( + (LongFunction) offset -> new MonthInterval(TZ, offset), + (Function) currentInterval -> currentInterval.plusMonths(1) + ), + Arguments.of( + (LongFunction) offset -> new QuarterInterval(TZ, offset), + (Function) currentInterval -> currentInterval.plusMonths(3) + ), + Arguments.of( + (LongFunction) offset -> new YearInterval(TZ, offset), + (Function) currentInterval -> currentInterval.plusYears(1) + ), + Arguments.of( + (LongFunction) offset -> new CustomInterval(TZ, offset, TimeUnit.HOURS.toSeconds(4)), + (Function) currentInterval -> currentInterval.plusHours(4) + ) + ); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index fa688a0a9e..c10da4e6c6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -138,7 +138,7 @@ public class CalculatedFieldDataValidator extends DataValidator if (minAggregationIntervalInSec <= 0) { return; } - if (aggConfiguration.getInterval().getIntervalDurationMillis() < TimeUnit.SECONDS.toMillis(minAggregationIntervalInSec)) { + if (aggConfiguration.getInterval().getCurrentIntervalDurationMillis() < TimeUnit.SECONDS.toMillis(minAggregationIntervalInSec)) { throw new IllegalArgumentException("Aggregation interval duration is less than configured " + "minimum allowed aggregation interval in tenant profile: " + minAggregationIntervalInSec + " sec."); } From 4c0cf4e0e6be1c4c9cdf5c4d011c6a10b1938ec9 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 12 Nov 2025 19:41:07 +0200 Subject: [PATCH 32/40] UI: Fixed entity aggregation cf not apply offset --- .../entity-aggregation-component.component.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts index ae63d0f7a5..5d612704a0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/entity-aggregation-configuration/entity-aggregation-component.component.ts @@ -38,7 +38,7 @@ import { import { filter, map } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { AVG_MONTH, AVG_QUARTER, DAY, HOUR, MINUTE, SECOND, YEAR } from '@shared/models/time/time.models'; -import { isDefinedAndNotNull } from '@core/utils'; +import { deepClone, isDefinedAndNotNull } from '@core/utils'; import { getCurrentAuthState } from '@core/auth/auth.selectors'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; @@ -156,7 +156,7 @@ export class EntityAggregationComponentComponent implements ControlValueAccessor this.entityAggregationConfiguration.valueChanges.pipe( takeUntilDestroyed() ).subscribe((value: CalculatedFieldEntityAggregationConfigurationValue) => { - this.updatedModel(value); + this.updatedModel(deepClone(value)); }); } From 1fd97498bea25c0192b6bea4b7fa4f5088748fd2 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 13 Nov 2025 10:42:44 +0200 Subject: [PATCH 33/40] handle tenant profile update --- .../server/actors/app/AppActor.java | 12 +++++++++ ...alculatedFieldManagerMessageProcessor.java | 19 ++++++++++++++ .../server/actors/tenant/TenantActor.java | 2 +- .../service/cf/CalculatedFieldCache.java | 2 ++ .../cf/DefaultCalculatedFieldCache.java | 5 ++++ .../cf/ctx/state/CalculatedFieldCtx.java | 26 ++++++++++++------- ...EntityAggregationCalculatedFieldState.java | 2 +- .../processing/AbstractConsumerService.java | 1 + .../thingsboard/server/cf/AlarmRulesTest.java | 3 ++- .../EntityAggregationCalculatedFieldTest.java | 5 ++-- 10 files changed, 63 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 20cacda26a..5a2f09f789 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -32,6 +32,7 @@ import org.thingsboard.server.actors.tenant.TenantActor; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.MsgType; @@ -165,6 +166,17 @@ public class AppActor extends ContextAwareActor { private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { TbActorRef target = null; if (TenantId.SYS_TENANT_ID.equals(msg.getTenantId())) { + if (msg.getEntityId() instanceof TenantProfileId tenantProfileId) { + tenantService.findTenantIdsByTenantProfileId(tenantProfileId).forEach(tenantId -> { + TbActorRef tenantActor = getOrCreateTenantActor(tenantId).orElseGet(() -> { + log.debug("Ignoring component lifecycle msg for tenant {} because it is not managed by this service", tenantId); + return null; + }); + if (tenantActor != null) { + tenantActor.tellWithHighPriority(msg); + } + }); + } if (!msg.getEntityId().getEntityType().isOneOf(EntityType.TENANT_PROFILE, EntityType.TB_RESOURCE)) { log.warn("Message has system tenant id: {}", msg); } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index a5b975d454..f0036b6627 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -71,6 +71,7 @@ import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -83,6 +84,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.function.Function; +import java.util.stream.Stream; import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; @@ -222,6 +224,12 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware default -> msg.getCallback().onSuccess(); } } + case TENANT_PROFILE -> { + switch (event) { + case UPDATED -> onTenantProfileUpdated(msg.getData(), msg.getCallback()); + default -> msg.getCallback().onSuccess(); + } + } default -> msg.getCallback().onSuccess(); } } @@ -247,6 +255,17 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware callback.onSuccess(); } + private void onTenantProfileUpdated(ComponentLifecycleMsg msg, TbCallback callback) { + Stream.concat( + calculatedFields.values().stream(), + entityIdCalculatedFields.values().stream().flatMap(Collection::stream) + ).forEach(CalculatedFieldCtx::updateTenantProfileProperties); + + calculatedFields.values().forEach(ctx -> { + applyToTargetCfEntityActors(ctx, callback, (id, cb) -> initCfForEntity(id, ctx, StateAction.REPROCESS, cb)); + }); + } + private void onEntityCreated(ComponentLifecycleMsg msg, TbCallback callback) { EntityId entityId = msg.getEntityId(); EntityId profileId = getProfileId(tenantId, entityId); diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 35a7f01b2e..11a8651026 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -350,7 +350,7 @@ public class TenantActor extends RuleChainManagerActor { } } if (cfActor != null) { - if (msg.getEntityId().getEntityType().isOneOf(EntityType.CALCULATED_FIELD, EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER)) { + if (msg.getEntityId().getEntityType().isOneOf(EntityType.CALCULATED_FIELD, EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT_PROFILE)) { cfActor.tellWithHighPriority(new CalculatedFieldEntityLifecycleMsg(tenantId, msg)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index d50a125451..5d643908ce 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -50,6 +50,8 @@ public interface CalculatedFieldCache { void evict(CalculatedFieldId calculatedFieldId); + void handleTenantProfileUpdate(); + EntityId getProfileId(TenantId tenantId, EntityId entityId); Set getDynamicEntities(TenantId tenantId, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 9e51997a20..1938e58a8f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -229,6 +229,11 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); } + @Override + public void handleTenantProfileUpdate() { + calculatedFieldsCtx.values().forEach(CalculatedFieldCtx::updateTenantProfileProperties); + } + @Override public EntityId getProfileId(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 518b65dc5f..8818ae542c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -97,9 +97,6 @@ public class CalculatedFieldCtx implements Closeable { private String expression; private boolean useLatestTs; - private long cfCheckInterval; - private long alarmReevaluationInterval; - private long lastReevaluationTs; private ActorSystemContext systemContext; @@ -113,7 +110,6 @@ public class CalculatedFieldCtx implements Closeable { private boolean initialized; - private long maxDataPointsPerRollingArg; private long maxStateSize; private long maxSingleValueArgumentSize; @@ -202,15 +198,12 @@ public class CalculatedFieldCtx implements Closeable { if (calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig) { this.useLatestTs = aggConfig.isUseLatestTs(); } - this.cfCheckInterval = systemContext.getCfCheckInterval(); - this.alarmReevaluationInterval = systemContext.getAlarmRulesReevaluationInterval(); this.systemContext = systemContext; this.tbelInvokeService = systemContext.getTbelInvokeService(); this.relationService = systemContext.getRelationService(); this.alarmService = systemContext.getAlarmService(); this.cfProcessingService = systemContext.getCalculatedFieldProcessingService(); - this.maxDataPointsPerRollingArg = systemContext.getApiLimitService().getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); // fixme why tenant profile update is not handled?? this.maxStateSize = systemContext.getApiLimitService().getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024; this.maxSingleValueArgumentSize = systemContext.getApiLimitService().getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxSingleValueArgumentSizeInKBytes) * 1024; } @@ -284,6 +277,11 @@ public class CalculatedFieldCtx implements Closeable { } } + public void updateTenantProfileProperties() { + this.maxStateSize = systemContext.getApiLimitService().getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024; + this.maxSingleValueArgumentSize = systemContext.getApiLimitService().getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxSingleValueArgumentSizeInKBytes) * 1024; + } + public double evaluateSimpleExpression(Expression expression, CalculatedFieldState state) { for (Map.Entry entry : state.getArguments().entrySet()) { try { @@ -645,9 +643,8 @@ public class CalculatedFieldCtx implements Closeable { if (calculatedField.getConfiguration() instanceof EntityAggregationCalculatedFieldConfiguration thisConfig && other.getCalculatedField().getConfiguration() instanceof EntityAggregationCalculatedFieldConfiguration otherConfig) { boolean metricsChanged = thisConfig.getMetrics().equals(otherConfig.getMetrics()); - boolean intervalChanged = thisConfig.getInterval().equals(otherConfig.getInterval()); boolean watermarkChanged = thisConfig.getWatermark().equals(otherConfig.getWatermark()); - return metricsChanged || intervalChanged || watermarkChanged; + return metricsChanged || watermarkChanged; } return false; } @@ -672,6 +669,9 @@ public class CalculatedFieldCtx implements Closeable { if (hasRelatedEntitiesAggregationConfigurationChanges(other)) { return true; } + if (hasEntityAggregationConfigurationChanges(other)) { + return true; + } return false; } @@ -691,6 +691,14 @@ public class CalculatedFieldCtx implements Closeable { return false; } + private boolean hasEntityAggregationConfigurationChanges(CalculatedFieldCtx other) { + if (calculatedField.getConfiguration() instanceof EntityAggregationCalculatedFieldConfiguration thisConfig + && other.calculatedField.getConfiguration() instanceof EntityAggregationCalculatedFieldConfiguration otherConfig) { + return !thisConfig.getInterval().equals(otherConfig.getInterval()); + } + return false; + } + private boolean isScheduledUpdateEnabled() { return scheduledUpdateIntervalMillis != -1; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java index a1208c5054..04179bdd31 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java @@ -69,7 +69,7 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt var configuration = (EntityAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); Watermark watermark = configuration.getWatermark(); watermarkDuration = watermark == null ? 0 : TimeUnit.SECONDS.toMillis(watermark.getDuration()); - checkInterval = TimeUnit.SECONDS.toMillis(ctx.getCfCheckInterval()); + checkInterval = TimeUnit.SECONDS.toMillis(ctx.getSystemContext().getCfCheckInterval()); interval = configuration.getInterval(); metrics = configuration.getMetrics(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 6e162256a4..37c3d31d0a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -166,6 +166,7 @@ public abstract class AbstractConsumerService { tenantProfileConfig.setMinAllowedDeduplicationIntervalInSecForCF(1); + tenantProfileConfig.setMinAllowedAggregationIntervalInSecForCF(1); }); Tenant tenant = new Tenant(); @@ -95,7 +96,7 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest public void testCreateCf_checkAggregation() throws Exception { Device device = createDevice("Device", "1234567890111"); - CustomInterval customInterval = new CustomInterval("Europe/Kyiv", 30L, 0L); + CustomInterval customInterval = new CustomInterval("Europe/Kyiv", 0L, 30L); long currentIntervalStartTs = customInterval.getCurrentIntervalStartTs(); long currentIntervalEndTs = customInterval.getCurrentIntervalEndTs(); @@ -126,7 +127,7 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest public void testCreateCf_checkAggregationDuringWatermark() throws Exception { Device device = createDevice("Device", "1234567890111"); - CustomInterval customInterval = new CustomInterval("Europe/Kyiv", 30L, 0L); + CustomInterval customInterval = new CustomInterval("Europe/Kyiv", 0L, 30L); long currentIntervalStartTs = customInterval.getCurrentIntervalStartTs(); long currentIntervalEndTs = customInterval.getCurrentIntervalEndTs(); From 0a6079f4cedf367c5446fdd3c489803b8d00caa0 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 13 Nov 2025 11:01:37 +0200 Subject: [PATCH 34/40] added refresh ctx action type --- .../CalculatedFieldEntityMessageProcessor.java | 10 ++++++---- .../CalculatedFieldManagerMessageProcessor.java | 2 +- .../calculatedField/EntityInitCalculatedFieldMsg.java | 3 ++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 5d9aeb17ff..b0476b0e34 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -159,10 +159,12 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else { state.setCtx(ctx, actorCtx); } - if (state.isSizeOk()) { - processStateIfReady(state, Collections.emptyMap(), ctx, Collections.singletonList(ctx.getCfId()), null, null, msg.getCallback()); - } else { - throw new RuntimeException(ctx.getSizeExceedsLimitMessage()); + if (msg.getStateAction() != StateAction.REFRESH_CTX) { + if (state.isSizeOk()) { + processStateIfReady(state, Collections.emptyMap(), ctx, Collections.singletonList(ctx.getCfId()), null, null, msg.getCallback()); + } else { + throw new RuntimeException(ctx.getSizeExceedsLimitMessage()); + } } } catch (Exception e) { log.debug("[{}][{}] Failed to initialize CF state", entityId, ctx.getCfId(), e); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index f0036b6627..75cf2f6748 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -262,7 +262,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware ).forEach(CalculatedFieldCtx::updateTenantProfileProperties); calculatedFields.values().forEach(ctx -> { - applyToTargetCfEntityActors(ctx, callback, (id, cb) -> initCfForEntity(id, ctx, StateAction.REPROCESS, cb)); + applyToTargetCfEntityActors(ctx, callback, (id, cb) -> initCfForEntity(id, ctx, StateAction.REFRESH_CTX, cb)); }); } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java index 1e0025988d..49f2c691d3 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java @@ -39,6 +39,7 @@ public class EntityInitCalculatedFieldMsg implements ToCalculatedFieldSystemMsg INIT, REINIT, RECREATE, - REPROCESS + REPROCESS, + REFRESH_CTX } } From 05dfbbf128139a40533b09b491fba6a0a7878db1 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 13 Nov 2025 11:52:52 +0200 Subject: [PATCH 35/40] process deleted telemetry --- .../service/cf/ctx/state/BaseCalculatedFieldState.java | 3 +++ .../single/EntityAggregationArgumentEntry.java | 8 +++++++- .../single/EntityAggregationCalculatedFieldState.java | 1 + .../aggregation/single/interval/CustomInterval.java | 4 ++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index e438274ab6..e8174967a5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.single.EntityAggregationArgumentEntry; import org.thingsboard.server.utils.CalculatedFieldUtils; import java.io.Closeable; @@ -82,6 +83,8 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, validateNewEntry(key, newEntry); if (existingEntry instanceof RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry) { relatedEntitiesArgumentEntry.updateEntry(newEntry); + } else if (existingEntry instanceof EntityAggregationArgumentEntry entityAggArgumentEntry) { + entityAggArgumentEntry.updateEntry(newEntry); } else { arguments.put(key, newEntry); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationArgumentEntry.java index d738c7d10b..7ec5098bc3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationArgumentEntry.java @@ -48,19 +48,25 @@ public class EntityAggregationArgumentEntry implements ArgumentEntry { @Override public boolean updateEntry(ArgumentEntry entry) { + boolean updated = false; if (entry instanceof EntityAggregationArgumentEntry entityAggEntry) { aggIntervals.putAll(entityAggEntry.getAggIntervals()); } else if (entry instanceof SingleValueArgumentEntry singleValueArgEntry) { long entryTs = singleValueArgEntry.getTs(); long argUpdateTs = System.currentTimeMillis(); for (Map.Entry aggIntervalEntry : aggIntervals.entrySet()) { + if (singleValueArgEntry.isForceResetPrevious()) { + aggIntervalEntry.getValue().setLastArgsRefreshTs(argUpdateTs); + updated = true; + continue; + } if (aggIntervalEntry.getKey().belongsToInterval(entryTs)) { aggIntervalEntry.getValue().setLastArgsRefreshTs(argUpdateTs); return true; } } } - return false; + return updated; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java index 04179bdd31..fbfdd9a258 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java @@ -188,6 +188,7 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt Map> results) { args.forEach((argName, argEntryIntervalStatus) -> { if (argEntryIntervalStatus.getLastArgsRefreshTs() > argEntryIntervalStatus.getLastMetricsEvalTs()) { + argEntryIntervalStatus.setLastMetricsEvalTs(System.currentTimeMillis()); processMetric(intervalEntry, argName, false, results); } else if (argEntryIntervalStatus.getLastMetricsEvalTs() == -1) { argEntryIntervalStatus.setLastMetricsEvalTs(System.currentTimeMillis()); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java index 39b2f59b15..c760de1143 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; @@ -28,6 +30,8 @@ import java.time.ZonedDateTime; @NoArgsConstructor public class CustomInterval extends BaseAggInterval { + @NotNull + @Min(1) private Long durationSec; public CustomInterval(String tz, Long offsetSec, Long durationSec) { From ed436c1dfa2db96021d715bbed97151c41658924 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 13 Nov 2025 13:47:03 +0200 Subject: [PATCH 36/40] fixed aligned method for custom interval --- .../aggregation/single/interval/CustomInterval.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java index c760de1143..a88769f223 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java @@ -55,10 +55,10 @@ public class CustomInterval extends BaseAggInterval { @Override protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { - long durationMillis = getDurationMillis(); - long nowMillis = reference.toInstant().toEpochMilli(); - long alignedStartMillis = (nowMillis / durationMillis) * durationMillis; - ZonedDateTime aligned = Instant.ofEpochMilli(alignedStartMillis).atZone(getZoneId()); + ZonedDateTime localMidnight = reference.toLocalDate().atStartOfDay(reference.getZone()); + long secondsFromMidnight = Duration.between(localMidnight, reference).getSeconds(); + long alignedSecondsFromMidnight = (secondsFromMidnight / durationSec) * durationSec; + ZonedDateTime aligned = localMidnight.plusSeconds(alignedSecondsFromMidnight); return next ? aligned.plusSeconds(durationSec) : aligned; } From 40e0d4a92f04b6cd3e3be92ae278e79a42da6c92 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 14 Nov 2025 16:02:45 +0200 Subject: [PATCH 37/40] added validation for config and tests --- .../EntityAggregationCalculatedFieldTest.java | 84 ++++++++---- .../CalculatedFieldControllerTest.java | 57 ++++++++ ...gregationCalculatedFieldConfiguration.java | 32 +++++ .../single/interval/AggInterval.java | 2 + .../single/interval/BaseAggInterval.java | 24 +++- ...ationCalculatedFieldConfigurationTest.java | 128 ++++++++++++++++++ .../single/interval/AggIntervalTest.java | 29 ++++ 7 files changed, 327 insertions(+), 29 deletions(-) create mode 100644 common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfigurationTest.java diff --git a/application/src/test/java/org/thingsboard/server/cf/EntityAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/EntityAggregationCalculatedFieldTest.java index 8e35b01cb5..e479c4959e 100644 --- a/application/src/test/java/org/thingsboard/server/cf/EntityAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/EntityAggregationCalculatedFieldTest.java @@ -93,41 +93,71 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest } @Test - public void testCreateCf_checkAggregation() throws Exception { + public void testCreateCfAndNoTelemetryDuringInterval_checkAggregation() throws Exception { Device device = createDevice("Device", "1234567890111"); - CustomInterval customInterval = new CustomInterval("Europe/Kyiv", 0L, 30L); + CustomInterval customInterval = new CustomInterval("Europe/Kyiv", 0L, 5L); + long intervalEndTs = customInterval.getCurrentIntervalEndTs(); + + CalculatedField totalConsumptionCF = createTotalConsumptionCF(device.getId(), customInterval, null); + long interval = customInterval.getCurrentIntervalDurationMillis(); + + await().alias("create CF and no telemetry during interval -> save metric with default value") + .atMost(2 * interval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode result = getLatestTelemetry(device.getId(), "consumption"); + assertThat(result).isNotNull(); + assertThat(result.get("consumption").get(0).get("value").asText()).isEqualTo("9999"); + }); + } + + @Test + public void testCreateCfWithoutWatermark_checkAggregation() throws Exception { + Device device = createDevice("Device", "1234567890111"); + + CustomInterval customInterval = new CustomInterval("Europe/Kyiv", 0L, 5L); long currentIntervalStartTs = customInterval.getCurrentIntervalStartTs(); long currentIntervalEndTs = customInterval.getCurrentIntervalEndTs(); - long tsBeforeInterval = currentIntervalStartTs - 1000L; - long tsInInterval_1 = currentIntervalStartTs + 1000L; - long tsInInterval_2 = currentIntervalStartTs + 500L; - long tsInInterval_3 = currentIntervalStartTs + 200L; + long tsBeforeInterval = currentIntervalStartTs - 1000; + long tsInInterval_1 = currentIntervalStartTs + 1000; + long tsInInterval_2 = currentIntervalStartTs + 500; + long tsInInterval_3 = currentIntervalStartTs + 200; postTelemetry(device.getId(), String.format("{\"ts\": \"%s\", \"values\": {\"energy\":120}}", tsBeforeInterval)); postTelemetry(device.getId(), String.format("{\"ts\": \"%s\", \"values\": {\"energy\":100}}", tsInInterval_1)); postTelemetry(device.getId(), String.format("{\"ts\": \"%s\", \"values\": {\"energy\":180}}", tsInInterval_2)); postTelemetry(device.getId(), String.format("{\"ts\": \"%s\", \"values\": {\"energy\":120}}", tsInInterval_3)); long interval = customInterval.getCurrentIntervalDurationMillis(); - Watermark watermark = new Watermark(60); - CalculatedField totalConsumptionCF = createTotalConsumptionCF(device.getId(), customInterval, watermark); + CalculatedField totalConsumptionCF = createTotalConsumptionCF(device.getId(), customInterval, null); + + await().alias("create CF -> perform aggregation after interval end") + .atMost(2 * interval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode result = getLatestTelemetry(device.getId(), "consumption"); + assertThat(result).isNotNull(); + assertThat(result.get("consumption").get(0).get("value").asText()).isEqualTo("400"); + }); + + postTelemetry(device.getId(), String.format("{\"ts\": \"%s\", \"values\": {\"energy\":500}}", tsInInterval_1)); - await().alias("create CF and perform aggregation after interval end") + await().alias("update telemetry that belongs to previous interval -> no aggregation since watermark is not set ") .atMost(2 * interval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode result = getLatestTelemetry(device.getId(), "consumptionPerMin"); + ObjectNode result = getLatestTelemetry(device.getId(), "consumption"); assertThat(result).isNotNull(); - assertThat(result.get("consumptionPerMin").get(0).get("value").asText()).isEqualTo("400"); + assertThat(result.get("consumption").get(0).get("value").asText()).isEqualTo("400"); }); } @Test - public void testCreateCf_checkAggregationDuringWatermark() throws Exception { + public void testCreateCfWithWatermark_checkAggregationDuringWatermark() throws Exception { Device device = createDevice("Device", "1234567890111"); - CustomInterval customInterval = new CustomInterval("Europe/Kyiv", 0L, 30L); + CustomInterval customInterval = new CustomInterval("Europe/Kyiv", 0L, 5L); long currentIntervalStartTs = customInterval.getCurrentIntervalStartTs(); long currentIntervalEndTs = customInterval.getCurrentIntervalEndTs(); @@ -141,27 +171,27 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest postTelemetry(device.getId(), String.format("{\"ts\": \"%s\", \"values\": {\"energy\":120}}", tsInInterval_3)); long interval = customInterval.getCurrentIntervalDurationMillis(); - Watermark watermark = new Watermark(60); + Watermark watermark = new Watermark(10); CalculatedField totalConsumptionCF = createTotalConsumptionCF(device.getId(), customInterval, watermark); - await().alias("create CF and perform aggregation after interval end") + await().alias("create CF -> perform aggregation after interval end") .atMost(2 * interval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode result = getLatestTelemetry(device.getId(), "consumptionPerMin"); + ObjectNode result = getLatestTelemetry(device.getId(), "consumption"); assertThat(result).isNotNull(); - assertThat(result.get("consumptionPerMin").get(0).get("value").asText()).isEqualTo("400"); + assertThat(result.get("consumption").get(0).get("value").asText()).isEqualTo("400"); }); postTelemetry(device.getId(), String.format("{\"ts\": \"%s\", \"values\": {\"energy\":300}}", tsInInterval_1)); - await().alias("create CF and perform aggregation after interval end") + await().alias("update telemetry during watermark -> perform aggregation") .atMost(2 * 10, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode result = getLatestTelemetry(device.getId(), "consumptionPerMin"); + ObjectNode result = getLatestTelemetry(device.getId(), "consumption"); assertThat(result).isNotNull(); - assertThat(result.get("consumptionPerMin").get(0).get("value").asText()).isEqualTo("600"); + assertThat(result.get("consumption").get(0).get("value").asText()).isEqualTo("600"); }); } @@ -169,15 +199,15 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest Map arguments = new HashMap<>(); Argument argument = new Argument(); argument.setRefEntityKey(new ReferencedEntityKey("energy", ArgumentType.TS_LATEST, null)); - argument.setLimit(100); arguments.put("en", argument); Map aggMetrics = new HashMap<>(); - AggMetric consumptionPerMin = new AggMetric(); - consumptionPerMin.setFunction(AggFunction.SUM); - consumptionPerMin.setInput(new AggKeyInput("en")); - aggMetrics.put("consumptionPerMin", consumptionPerMin); + AggMetric consumption = new AggMetric(); + consumption.setFunction(AggFunction.SUM); + consumption.setInput(new AggKeyInput("en")); + consumption.setDefaultValue(9999L); + aggMetrics.put("consumption", consumption); Output output = new Output(); output.setType(OutputType.TIME_SERIES); @@ -208,7 +238,9 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest configuration.setArguments(inputs); configuration.setMetrics(metrics); configuration.setInterval(aggInterval); - configuration.setWatermark(watermark); + if (watermark != null) { + configuration.setWatermark(watermark); + } configuration.setOutput(output); calculatedField.setConfiguration(configuration); diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 4ebace6ae7..61fc7a9e48 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -32,6 +32,11 @@ import org.thingsboard.server.common.data.cf.configuration.PropagationCalculated import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.EntityAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.HourInterval; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.Watermark; import org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.ZoneGroupConfiguration; @@ -45,6 +50,7 @@ import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; @@ -166,6 +172,34 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { .andExpect(status().isOk()); } + @Test + public void testSaveEntityAggregationCalculatedField() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId(), CalculatedFieldType.ENTITY_AGGREGATION); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + assertThat(savedCalculatedField).isNotNull(); + assertThat(savedCalculatedField.getId()).isNotNull(); + assertThat(savedCalculatedField.getCreatedTime()).isGreaterThan(0); + assertThat(savedCalculatedField.getTenantId()).isEqualTo(savedTenant.getId()); + assertThat(savedCalculatedField.getEntityId()).isEqualTo(calculatedField.getEntityId()); + assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); + assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(savedCalculatedField.getConfiguration()).isEqualTo(getEntityAggregationCalculatedFieldConfig()); + assertThat(savedCalculatedField.getVersion()).isEqualTo(1L); + + savedCalculatedField.setName("Test CF"); + + CalculatedField updatedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + assertThat(updatedCalculatedField.getName()).isEqualTo(savedCalculatedField.getName()); + assertThat(updatedCalculatedField.getVersion()).isEqualTo(savedCalculatedField.getVersion() + 1); + + doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) + .andExpect(status().isOk()); + } + @Test public void testSavePropagationCalculatedFieldWithNullArguments() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); @@ -237,6 +271,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { case SIMPLE -> calculatedField.setConfiguration(getSimpleCalculatedFieldConfig()); case GEOFENCING -> calculatedField.setConfiguration(getGeofencingCalculatedFieldConfig()); case PROPAGATION -> calculatedField.setConfiguration(getPropagationCalculatedFieldConfig()); + case ENTITY_AGGREGATION -> calculatedField.setConfiguration(getEntityAggregationCalculatedFieldConfig()); } calculatedField.setVersion(1L); return calculatedField; @@ -287,6 +322,28 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { return config; } + private CalculatedFieldConfiguration getEntityAggregationCalculatedFieldConfig() { + var config = new EntityAggregationCalculatedFieldConfiguration(); + + Argument energyArgument = new Argument(); + energyArgument.setRefEntityKey(new ReferencedEntityKey("energy", ArgumentType.TS_LATEST, null)); + config.setArguments(Map.of("en", energyArgument)); + + AggMetric metric = new AggMetric(); + metric.setInput(new AggKeyInput("en")); + metric.setDefaultValue(9999L); + config.setMetrics(Map.of("consumption", metric)); + + config.setWatermark(new Watermark(TimeUnit.DAYS.toSeconds(1))); + config.setInterval(new HourInterval("Europe/Kiev", TimeUnit.MINUTES.toSeconds(15))); + + Output output = new Output(); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + return config; + } + private CalculatedFieldConfiguration getSimpleCalculatedFieldConfig() { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java index eb97538701..f6095d41a7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfiguration.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ArgumentsBasedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.AggInterval; import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.Watermark; @@ -53,6 +54,12 @@ public class EntityAggregationCalculatedFieldConfiguration implements ArgumentsB @Override public void validate() { + validateArguments(); + validateMetrics(); + validateInterval(); + } + + private void validateArguments() { if (arguments.containsKey("ctx")) { throw new IllegalArgumentException("Argument name 'ctx' is reserved and cannot be used."); } @@ -61,4 +68,29 @@ public class EntityAggregationCalculatedFieldConfiguration implements ArgumentsB } } + private void validateMetrics() { + if (metrics == null || metrics.isEmpty()) { + throw new IllegalArgumentException("Metrics map cannot be empty."); + } + + for (AggMetric metric : metrics.values()) { + if (metric.getInput() instanceof AggKeyInput aggKeyInput) { + if (!arguments.containsKey(aggKeyInput.getKey())) { + throw new IllegalArgumentException( + "Metric references unknown argument: '" + aggKeyInput.getKey() + "'." + ); + } + } else { + throw new IllegalArgumentException("Metric key can only refer to argument."); + } + } + } + + private void validateInterval() { + if (interval == null) { + throw new IllegalArgumentException("Interval must be defined."); + } + interval.validate(); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java index 6bff7b4398..3d38ebc1f6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggInterval.java @@ -62,4 +62,6 @@ public interface AggInterval { ZonedDateTime getNextIntervalStart(ZonedDateTime currentStart); + void validate(); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java index 53f7117bfc..e6400230b3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java @@ -23,6 +23,7 @@ import lombok.NoArgsConstructor; import java.time.ZoneId; import java.time.ZonedDateTime; +import java.util.concurrent.TimeUnit; @Data @JsonInclude(JsonInclude.Include.NON_NULL) @@ -39,7 +40,7 @@ public abstract class BaseAggInterval implements AggInterval { return ZoneId.of(tz); } - protected long getOffset() { + protected long getOffsetSafe() { return offsetSec != null ? offsetSec : 0L; } @@ -57,7 +58,7 @@ public abstract class BaseAggInterval implements AggInterval { @Override public long getDateTimeIntervalStartTs(ZonedDateTime dateTime) { - long offset = getOffset(); + long offset = getOffsetSafe(); ZonedDateTime shiftedNow = dateTime.minusSeconds(offset); ZonedDateTime alignedStart = getAlignedBoundary(shiftedNow, false); ZonedDateTime actualStart = alignedStart.plusSeconds(offset); @@ -73,7 +74,7 @@ public abstract class BaseAggInterval implements AggInterval { @Override public long getDateTimeIntervalEndTs(ZonedDateTime dateTime) { - long offset = getOffset(); + long offset = getOffsetSafe(); ZonedDateTime shiftedNow = dateTime.minusSeconds(offset); ZonedDateTime alignedEnd = getAlignedBoundary(shiftedNow, true); ZonedDateTime actualEnd = alignedEnd.plusSeconds(offset); @@ -82,4 +83,21 @@ public abstract class BaseAggInterval implements AggInterval { protected abstract ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next); + @Override + public void validate() { + try { + getZoneId(); + } catch (Exception ex) { + throw new IllegalArgumentException("Invalid timezone in interval: " + ex.getMessage()); + } + if (offsetSec != null) { + if (offsetSec < 0) { + throw new IllegalArgumentException("Offset cannot be negative."); + } + if (TimeUnit.SECONDS.toMillis(offsetSec) >= getCurrentIntervalDurationMillis()) { + throw new IllegalArgumentException("Offset must be greater than interval duration."); + } + } + } + } diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfigurationTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfigurationTest.java new file mode 100644 index 0000000000..3884b5a214 --- /dev/null +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/EntityAggregationCalculatedFieldConfigurationTest.java @@ -0,0 +1,128 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation.single; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; +import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.HourInterval; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class EntityAggregationCalculatedFieldConfigurationTest { + + @Test + void typeShouldBeEntityAggregation() { + var cfg = new EntityAggregationCalculatedFieldConfiguration(); + assertThat(cfg.getType()).isEqualTo(CalculatedFieldType.ENTITY_AGGREGATION); + } + + @ParameterizedTest + @ValueSource(strings = {"ATTRIBUTE", "TS_ROLLING"}) + void validateShouldThrowWhenNotTsLatestArgumentUsed(String argumentType) { + var cfg = new EntityAggregationCalculatedFieldConfiguration(); + cfg.setArguments(Map.of("k", validArgument(ArgumentType.valueOf(argumentType)))); + assertThatThrownBy(cfg::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Calculated field with type: '" + cfg.getType() + "' support only TS_LATEST arguments."); + } + + @Test + void validateShouldThrowWhenMetricMapIsEmpty() { + var cfg = new EntityAggregationCalculatedFieldConfiguration(); + + cfg.setArguments(Map.of("k", validArgument(ArgumentType.TS_LATEST))); + cfg.setMetrics(Map.of()); + + assertThatThrownBy(cfg::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Metrics map cannot be empty."); + } + + @Test + void validateShouldThrowWhenMetricInputIsNotAggKeyInput() { + var cfg = new EntityAggregationCalculatedFieldConfiguration(); + + cfg.setArguments(Map.of("k", validArgument(ArgumentType.TS_LATEST))); + + AggMetric metric = new AggMetric(); + metric.setInput(new AggFunctionInput()); // cannot be function + cfg.setMetrics(Map.of("m", metric)); + + cfg.setInterval(new HourInterval("Europe/Kiev", null)); + cfg.setOutput(new Output()); + + assertThatThrownBy(cfg::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Metric key can only refer to argument."); + } + + @Test + void validateShouldThrowWhenMetricReferencesUnknownArgument() { + var cfg = new EntityAggregationCalculatedFieldConfiguration(); + + cfg.setArguments(Map.of("k", validArgument(ArgumentType.TS_LATEST))); + + AggMetric metric = new AggMetric(); + metric.setInput(new AggKeyInput("unknown")); + cfg.setMetrics(Map.of("m", metric)); + + cfg.setInterval(new HourInterval("Europe/Kiev", null)); + cfg.setOutput(new Output()); + + assertThatThrownBy(cfg::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Metric references unknown argument: 'unknown'."); + } + + @Test + void validateShouldThrowWhenIntervalIsNull() { + var cfg = new EntityAggregationCalculatedFieldConfiguration(); + + cfg.setArguments(Map.of("k", validArgument(ArgumentType.TS_LATEST))); + cfg.setMetrics(Map.of("m", validMetric())); + cfg.setInterval(null); + cfg.setOutput(new Output()); + + assertThatThrownBy(cfg::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Interval must be defined."); + } + + private Argument validArgument(ArgumentType type) { + Argument a = new Argument(); + a.setRefEntityKey(new ReferencedEntityKey("key", type, null)); + return a; + } + + private AggMetric validMetric() { + AggMetric metric = new AggMetric(); + metric.setInput(new AggKeyInput("k")); + return metric; + } + +} diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalTest.java index 38fda87879..b439c44fef 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/AggIntervalTest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -29,11 +30,39 @@ import java.util.function.LongFunction; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class AggIntervalTest { private static final String TZ = "Europe/Kiev"; + @Test + void validateShouldThrowWhenInvalidTimZone() { + AggInterval interval = new HourInterval("TimeZone", null); + + assertThatThrownBy(interval::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid timezone in interval: "); + } + + @Test + void validateShouldThrowWhenOffsetIsNegative() { + AggInterval interval = new CustomInterval(TZ, -100L, TimeUnit.HOURS.toSeconds(2)); + + assertThatThrownBy(interval::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Offset cannot be negative."); + } + + @Test + void validateShouldThrowWhenOffsetGreaterThanIntervalDuration() { + AggInterval interval = new CustomInterval(TZ, TimeUnit.HOURS.toSeconds(2), TimeUnit.HOURS.toSeconds(2)); + + assertThatThrownBy(interval::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Offset must be greater than interval duration."); + } + @ParameterizedTest @MethodSource("intervals") void testGetStartAndEndWithoutOffset(LongFunction intervalCreator) { From 7b9b15d0f2a27fd91eaa06243586b4a3d679c708 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 17 Nov 2025 17:06:21 +0200 Subject: [PATCH 38/40] review fixes --- .../server/actors/app/AppActor.java | 19 ++++++++++--------- ...CalculatedFieldEntityMessageProcessor.java | 13 +++++-------- ...alculatedFieldManagerMessageProcessor.java | 5 +---- .../EntityInitCalculatedFieldMsg.java | 3 +-- .../cf/ctx/state/CalculatedFieldCtx.java | 16 +++++++++------- ...EntityAggregationCalculatedFieldState.java | 6 +++--- .../alarm/AlarmCalculatedFieldState.java | 3 --- .../src/main/resources/thingsboard.yml | 2 +- 8 files changed, 30 insertions(+), 37 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 5a2f09f789..da16e55db8 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -166,16 +166,17 @@ public class AppActor extends ContextAwareActor { private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { TbActorRef target = null; if (TenantId.SYS_TENANT_ID.equals(msg.getTenantId())) { - if (msg.getEntityId() instanceof TenantProfileId tenantProfileId) { - tenantService.findTenantIdsByTenantProfileId(tenantProfileId).forEach(tenantId -> { - TbActorRef tenantActor = getOrCreateTenantActor(tenantId).orElseGet(() -> { - log.debug("Ignoring component lifecycle msg for tenant {} because it is not managed by this service", tenantId); - return null; + if (systemContext.isTenantComponentsInitEnabled()) { + if (msg.getEntityId() instanceof TenantProfileId tenantProfileId) { + tenantService.findTenantIdsByTenantProfileId(tenantProfileId).forEach(tenantId -> { + getOrCreateTenantActor(tenantId).ifPresentOrElse(tenantActor -> { + log.debug("[{}] Sending component lifecycle msg for tenant.", tenantId); + tenantActor.tellWithHighPriority(msg); + }, () -> { + log.debug("Ignoring component lifecycle msg for tenant {} because it is not managed by this service", tenantId); + }); }); - if (tenantActor != null) { - tenantActor.tellWithHighPriority(msg); - } - }); + } } if (!msg.getEntityId().getEntityType().isOneOf(EntityType.TENANT_PROFILE, EntityType.TB_RESOURCE)) { log.warn("Message has system tenant id: {}", msg); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index b0476b0e34..bf9a3529e5 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -123,7 +123,6 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (state != null) { state.setCtx(msg.getCtx(), actorCtx); state.setPartition(msg.getPartition()); - state.init(true); states.put(cfId, state); } else { removeState(cfId); @@ -134,7 +133,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM log.debug("Processing CF state partition restore msg: {}", msg); for (CalculatedFieldState state : states.values()) { if (msg.getPartition().equals(state.getPartition())) { - state.init(false); + state.init(true); } } } @@ -159,12 +158,10 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else { state.setCtx(ctx, actorCtx); } - if (msg.getStateAction() != StateAction.REFRESH_CTX) { - if (state.isSizeOk()) { - processStateIfReady(state, Collections.emptyMap(), ctx, Collections.singletonList(ctx.getCfId()), null, null, msg.getCallback()); - } else { - throw new RuntimeException(ctx.getSizeExceedsLimitMessage()); - } + if (state.isSizeOk()) { + processStateIfReady(state, Collections.emptyMap(), ctx, Collections.singletonList(ctx.getCfId()), null, null, msg.getCallback()); + } else { + throw new RuntimeException(ctx.getSizeExceedsLimitMessage()); } } catch (Exception e) { log.debug("[{}][{}] Failed to initialize CF state", entityId, ctx.getCfId(), e); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 75cf2f6748..ef0abff4dc 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -260,10 +260,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware calculatedFields.values().stream(), entityIdCalculatedFields.values().stream().flatMap(Collection::stream) ).forEach(CalculatedFieldCtx::updateTenantProfileProperties); - - calculatedFields.values().forEach(ctx -> { - applyToTargetCfEntityActors(ctx, callback, (id, cb) -> initCfForEntity(id, ctx, StateAction.REFRESH_CTX, cb)); - }); + callback.onSuccess(); } private void onEntityCreated(ComponentLifecycleMsg msg, TbCallback callback) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java index 49f2c691d3..1e0025988d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java @@ -39,7 +39,6 @@ public class EntityInitCalculatedFieldMsg implements ToCalculatedFieldSystemMsg INIT, REINIT, RECREATE, - REPROCESS, - REFRESH_CTX + REPROCESS } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 8818ae542c..1d2e2f505a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -221,16 +221,18 @@ public class CalculatedFieldCtx implements Closeable { return true; } } - long reevaluationIntervalMillis = TimeUnit.SECONDS.toMillis(systemContext.getAlarmRulesReevaluationInterval()); boolean requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); - if (requiresScheduledReevaluation) { - if (now + cfCheckIntervalMillis >= lastReevaluationTs + reevaluationIntervalMillis) { - lastReevaluationTs = now; - return true; + if (calculatedField.getConfiguration() instanceof AlarmCalculatedFieldConfiguration) { + long reevaluationIntervalMillis = TimeUnit.SECONDS.toMillis(systemContext.getAlarmRulesReevaluationInterval()); + if (requiresScheduledReevaluation) { + if (now + cfCheckIntervalMillis >= lastReevaluationTs + reevaluationIntervalMillis) { + lastReevaluationTs = now; + return true; + } + return false; } - return false; } - return false; + return requiresScheduledReevaluation; } public void init() { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java index fbfdd9a258..b07600695c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java @@ -127,7 +127,7 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt }); } - public void fillMissingIntervals() { + private void fillMissingIntervals() { ZoneId zoneId = interval.getZoneId(); long currentIntervalEndTs = interval.getCurrentIntervalEndTs(); @@ -253,12 +253,12 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt metricsNode.put(metricName, JacksonUtil.toString(resultValue)); } } - ObjectNode resultNode = JacksonUtil.newObjectNode(); if (!metricsNode.isEmpty()) { + ObjectNode resultNode = JacksonUtil.newObjectNode(); resultNode.put("ts", interval.getEndTs() - 1); resultNode.set("values", metricsNode); + result.add(resultNode); } - result.add(resultNode); }); return result; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java index 83e08ea67f..342f7534c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java @@ -124,9 +124,6 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState { @Override public void init(boolean restored) { super.init(restored); - if (restored) { - return; - } AtomicBoolean reevalNeeded = new AtomicBoolean(false); Map createRules = configuration.getCreateRules(); for (AlarmSeverity severity : AlarmSeverity.values()) { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index cdeb55b7b7..b1d66b2f25 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -529,7 +529,7 @@ actors: configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" # Time in seconds to receive calculation result. calculation_timeout: "${ACTORS_CALCULATION_TIMEOUT_SEC:5}" - # Interval in seconds to re-evaluate calculated fields that have a time schedule. 1 minute by default. + # Interval in seconds to check calculated fields for re-evaluation interval. 1 minute by default. check_interval: "${ACTORS_CALCULATED_FIELDS_CHECK_INTERVAL_SEC:60}" alarms: # Interval in seconds to re-evaluate Alarm rules that have a time schedule. 2 minutes by default. From 097bfb026654e9c8daf4c2d29d6752e8b6ffeb79 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 18 Nov 2025 08:20:52 +0200 Subject: [PATCH 39/40] refactoring --- .../aggregation/single/interval/BaseAggInterval.java | 7 ++++++- .../aggregation/single/interval/CustomInterval.java | 6 ++---- .../aggregation/single/interval/DayInterval.java | 5 ++--- .../aggregation/single/interval/HourInterval.java | 5 ++--- .../aggregation/single/interval/MonthInterval.java | 5 ++--- .../aggregation/single/interval/QuarterInterval.java | 5 ++--- .../aggregation/single/interval/WeekInterval.java | 6 ++---- .../aggregation/single/interval/WeekSunSatInterval.java | 6 ++---- .../aggregation/single/interval/YearInterval.java | 5 ++--- 9 files changed, 22 insertions(+), 28 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java index e6400230b3..0c0801dea9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/BaseAggInterval.java @@ -81,7 +81,12 @@ public abstract class BaseAggInterval implements AggInterval { return actualEnd.toInstant().toEpochMilli(); } - protected abstract ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next); + protected abstract ZonedDateTime alignToIntervalStart(ZonedDateTime reference); + + protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { + ZonedDateTime base = alignToIntervalStart(reference); + return next ? getNextIntervalStart(base) : base; + } @Override public void validate() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java index a88769f223..24bfa26d8d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/CustomInterval.java @@ -22,7 +22,6 @@ import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import java.time.Duration; -import java.time.Instant; import java.time.ZonedDateTime; @EqualsAndHashCode(callSuper = true) @@ -54,12 +53,11 @@ public class CustomInterval extends BaseAggInterval { } @Override - protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { + protected ZonedDateTime alignToIntervalStart(ZonedDateTime reference) { ZonedDateTime localMidnight = reference.toLocalDate().atStartOfDay(reference.getZone()); long secondsFromMidnight = Duration.between(localMidnight, reference).getSeconds(); long alignedSecondsFromMidnight = (secondsFromMidnight / durationSec) * durationSec; - ZonedDateTime aligned = localMidnight.plusSeconds(alignedSecondsFromMidnight); - return next ? aligned.plusSeconds(durationSec) : aligned; + return localMidnight.plusSeconds(alignedSecondsFromMidnight); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java index 37e75c9ee6..2ad0ce24d3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/DayInterval.java @@ -35,9 +35,8 @@ public class DayInterval extends BaseAggInterval { } @Override - protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { - ZonedDateTime base = reference.truncatedTo(ChronoUnit.DAYS); - return next ? base.plusDays(1) : base; + protected ZonedDateTime alignToIntervalStart(ZonedDateTime reference) { + return reference.truncatedTo(ChronoUnit.DAYS); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java index 1cac0017e7..3ce366bc75 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/HourInterval.java @@ -37,9 +37,8 @@ public class HourInterval extends BaseAggInterval { } @Override - protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { - ZonedDateTime base = reference.truncatedTo(ChronoUnit.HOURS); - return next ? base.plusHours(1) : base; + protected ZonedDateTime alignToIntervalStart(ZonedDateTime reference) { + return reference.truncatedTo(ChronoUnit.HOURS); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java index 0a540e49cd..3ce121459b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/MonthInterval.java @@ -35,9 +35,8 @@ public class MonthInterval extends BaseAggInterval { } @Override - protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { - ZonedDateTime base = reference.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS); - return next ? base.plusMonths(1) : base; + protected ZonedDateTime alignToIntervalStart(ZonedDateTime reference) { + return reference.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/QuarterInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/QuarterInterval.java index bd27c681f8..96fcfc0dc8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/QuarterInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/QuarterInterval.java @@ -36,14 +36,13 @@ public class QuarterInterval extends BaseAggInterval { } @Override - protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { + protected ZonedDateTime alignToIntervalStart(ZonedDateTime reference) { int month = reference.getMonthValue(); int quarterStartMonth = ((month - 1) / 3) * 3 + 1; // 1, 4, 7, 10 - ZonedDateTime base = ZonedDateTime.of( + return ZonedDateTime.of( LocalDate.of(reference.getYear(), quarterStartMonth, 1), LocalTime.MIDNIGHT, reference.getZone()); - return next ? base.plusMonths(3) : base; } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java index 381fb3bb66..0f70adc5ad 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekInterval.java @@ -37,10 +37,8 @@ public class WeekInterval extends BaseAggInterval { } @Override - protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { - ZonedDateTime startOfWeekDate = reference.with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)) - .truncatedTo(ChronoUnit.DAYS); - return next ? startOfWeekDate.plusWeeks(1) : startOfWeekDate; + protected ZonedDateTime alignToIntervalStart(ZonedDateTime reference) { + return reference.with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)).truncatedTo(ChronoUnit.DAYS); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java index 242f3b7914..2c4482f0b4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/WeekSunSatInterval.java @@ -37,10 +37,8 @@ public class WeekSunSatInterval extends BaseAggInterval { } @Override - protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { - ZonedDateTime startOfWeekDate = reference.with(TemporalAdjusters.previousOrSame(DayOfWeek.SUNDAY)) - .truncatedTo(ChronoUnit.DAYS); - return next ? startOfWeekDate.plusWeeks(1) : startOfWeekDate; + protected ZonedDateTime alignToIntervalStart(ZonedDateTime reference) { + return reference.with(TemporalAdjusters.previousOrSame(DayOfWeek.SUNDAY)).truncatedTo(ChronoUnit.DAYS); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java index 83c8f58301..441f3913a9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/single/interval/YearInterval.java @@ -36,12 +36,11 @@ public class YearInterval extends BaseAggInterval { } @Override - protected ZonedDateTime getAlignedBoundary(ZonedDateTime reference, boolean next) { - ZonedDateTime base = ZonedDateTime.of( + protected ZonedDateTime alignToIntervalStart(ZonedDateTime reference) { + return ZonedDateTime.of( LocalDate.of(reference.getYear(), 1, 1), LocalTime.MIDNIGHT, reference.getZone()); - return next ? base.plusYears(1) : base; } @Override From c0f4bba52c3589bdce1cd8e855517d0fdb2ea5b4 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 18 Nov 2025 10:11:33 +0200 Subject: [PATCH 40/40] passed tenant profile id to filter unnecessary cfs to update --- .../server/service/cf/CalculatedFieldCache.java | 3 ++- .../service/cf/DefaultCalculatedFieldCache.java | 13 +++++++++++-- .../queue/processing/AbstractConsumerService.java | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index 5d643908ce..54ddedcc42 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.List; @@ -50,7 +51,7 @@ public interface CalculatedFieldCache { void evict(CalculatedFieldId calculatedFieldId); - void handleTenantProfileUpdate(); + void handleTenantProfileUpdate(TenantProfileId tenantProfileId); EntityId getProfileId(TenantId tenantId, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 24c4b24a15..f2b8d4d9db 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service; import org.springframework.util.ConcurrentReferenceHashMap; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -33,8 +34,10 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -61,6 +64,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final CalculatedFieldService calculatedFieldService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; + private final TbTenantProfileCache tenantProfileCache; @Lazy private final ActorSystemContext systemContext; private final OwnerService ownerService; @@ -228,8 +232,13 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } @Override - public void handleTenantProfileUpdate() { - calculatedFieldsCtx.values().forEach(CalculatedFieldCtx::updateTenantProfileProperties); + public void handleTenantProfileUpdate(TenantProfileId tenantProfileId) { + calculatedFieldsCtx.values().stream() + .filter(ctx -> { + TenantProfile tenantProfile = tenantProfileCache.get(ctx.getTenantId()); + return tenantProfile != null && tenantProfileId.equals(tenantProfile.getId()); + }) + .forEach(CalculatedFieldCtx::updateTenantProfileProperties); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 37c3d31d0a..fa6c280cb7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -166,7 +166,7 @@ public abstract class AbstractConsumerService