From f667f0f23f796492050e336f1fee37e8495a69d0 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 10 Oct 2025 16:58:00 +0300 Subject: [PATCH 01/33] (wip) aggregation cf --- .../CalculatedFieldEntityActor.java | 3 + ...CalculatedFieldEntityMessageProcessor.java | 154 +++++++-- ...alculatedFieldManagerMessageProcessor.java | 320 +++++++++++++++++- .../CalculatedFieldRelatedEntityMsg.java | 49 +++ ...tractCalculatedFieldProcessingService.java | 97 ++++++ .../service/cf/CalculatedFieldCache.java | 3 + .../cf/CalculatedFieldProcessingService.java | 2 + .../cf/DefaultCalculatedFieldCache.java | 28 ++ ...faultCalculatedFieldProcessingService.java | 8 + .../DefaultCalculatedFieldQueueService.java | 39 ++- .../service/cf/ctx/state/ArgumentEntry.java | 10 +- .../cf/ctx/state/ArgumentEntryType.java | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 2 +- .../cf/ctx/state/CalculatedFieldCtx.java | 140 ++++++++ .../cf/ctx/state/CalculatedFieldState.java | 5 +- .../ctx/state/SingleValueArgumentEntry.java | 8 +- .../state/aggregation/AggArgumentEntry.java | 72 ++++ .../aggregation/AggSingleArgumentEntry.java | 83 +++++ ...ValuesAggregationCalculatedFieldState.java | 161 +++++++++ .../service/cf/ctx/state/aggregation/agg.json | 65 ++++ .../state/aggregation/function/AggEntry.java | 45 +++ .../function/AggFunctionFactory.java | 33 ++ .../aggregation/function/AvgAggEntry.java | 45 +++ .../aggregation/function/BaseAggEntry.java | 52 +++ .../aggregation/function/CountAggEntry.java | 41 +++ .../function/CountUniqueAggEntry.java | 45 +++ .../aggregation/function/MaxAggEntry.java | 40 +++ .../aggregation/function/MinAggEntry.java | 40 +++ .../aggregation/function/SumAggEntry.java | 42 +++ .../entitiy/EntityStateSourcingListener.java | 11 + .../queue/DefaultTbClusterService.java | 40 ++- .../processing/AbstractConsumerService.java | 18 +- .../utils/CalculatedFieldArgumentUtils.java | 2 + .../server/utils/CalculatedFieldUtils.java | 48 ++- ...tValuesAggregationCalculatedFieldTest.java | 314 +++++++++++++++++ .../server/controller/AbstractWebTest.java | 10 + .../server/cluster/TbClusterService.java | 8 + .../server/dao/relation/RelationService.java | 8 + .../common/data/cf/CalculatedFieldType.java | 3 +- .../CalculatedFieldConfiguration.java | 4 +- .../aggregation/AggFunction.java | 20 ++ .../aggregation/AggFunctionInput.java | 34 ++ .../configuration/aggregation/AggInput.java | 36 ++ .../aggregation/AggKeyInput.java | 34 ++ .../configuration/aggregation/AggMetric.java | 31 ++ .../configuration/aggregation/AggSource.java | 30 ++ .../aggregation/CfAggTrigger.java | 104 ++++++ ...gregationCalculatedFieldConfiguration.java | 52 +++ .../server/common/msg/MsgType.java | 2 + .../msg/plugin/ComponentLifecycleMsg.java | 6 +- .../server/common/util/ProtoUtils.java | 2 + common/proto/src/main/proto/queue.proto | 7 + .../script/api/tbel/TbelCfArg.java | 1 + .../tbel/TbelCfLatestValuesAggregation.java | 44 +++ .../dao/relation/BaseRelationService.java | 34 ++ .../server/dao/relation/RelationCacheKey.java | 7 +- .../server/dao/relation/RelationDao.java | 4 + .../dao/sql/relation/JpaRelationDao.java | 10 + .../dao/sql/relation/RelationRepository.java | 39 ++- 59 files changed, 2527 insertions(+), 70 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java create mode 100644 application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunction.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunctionInput.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggKeyInput.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java index cababd4b6d..ed4131c114 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -73,6 +73,9 @@ public class CalculatedFieldEntityActor extends AbstractCalculatedFieldActor { case CF_ENTITY_DELETE_MSG: processor.process((CalculatedFieldEntityDeleteMsg) msg); break; + case CF_RELATED_ENTITY_MSG: + processor.process((CalculatedFieldRelatedEntityMsg) msg); + break; case CF_ENTITY_TELEMETRY_MSG: processor.process((EntityCalculatedFieldTelemetryMsg) msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index ccbdcc6b33..98b5eba4c8 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -52,6 +52,9 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -66,7 +69,9 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createStateByType; @@ -189,7 +194,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } - public void process(CalculatedFieldEntityDeleteMsg msg) { + public void process(CalculatedFieldEntityDeleteMsg msg) throws CalculatedFieldException { log.debug("[{}] Processing CF entity delete msg.", msg.getEntityId()); if (this.entityId.equals(msg.getEntityId())) { if (states.isEmpty()) { @@ -200,16 +205,74 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM actorCtx.stop(actorCtx.getSelf()); } } else { - var cfId = new CalculatedFieldId(msg.getEntityId().getId()); - var state = removeState(cfId); - if (state != null) { - cfStateService.deleteState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); + EntityId msgEntityId = msg.getEntityId(); + if (msgEntityId instanceof CalculatedFieldId cfId) { + var state = removeState(cfId); + if (state != null) { + cfStateService.deleteState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); + } else { + msg.getCallback().onSuccess(); + } } else { - msg.getCallback().onSuccess(); + if (states.isEmpty()) { + msg.getCallback().onSuccess(); + } + for (Map.Entry entry : states.entrySet()) { + LatestValuesAggregationCalculatedFieldState state = (LatestValuesAggregationCalculatedFieldState) entry.getValue(); + state.getArguments().forEach((argName, argEntry) -> { + AggArgumentEntry aggArgEntry = (AggArgumentEntry) argEntry; + aggArgEntry.getAggInputs().remove(msgEntityId); + }); + state.getInputs().remove(msgEntityId); + state.setLastMetricsEvalTs(-1); + processStateIfReady(state, Collections.emptyMap(), state.getCtx(), Collections.emptyList(), null, null, msg.getCallback()); + } } } } + public void process(CalculatedFieldRelatedEntityMsg msg) throws CalculatedFieldException { + log.debug("[{}] Processing CF related entity msg.", msg.getEntityId()); + CalculatedFieldCtx cfCtx = msg.getCalculatedField(); + var state = states.get(cfCtx.getCfId()); + Map fetchedArguments = fetchAggArguments(msg.getCalculatedField(), msg.getEntityId()); + try { + if (state == null) { + state = createState(cfCtx); + } else { + state.setCtx(cfCtx, actorCtx); + } + if (state.isSizeOk()) { + if (state instanceof LatestValuesAggregationCalculatedFieldState latestValuesState) { + latestValuesState.setLastMetricsEvalTs(-1); + } + state.update(fetchedArguments, cfCtx); + state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, cfCtx.getCfId(), entityId), cfCtx.getMaxStateSize()); + states.put(cfCtx.getCfId(), state); + processStateIfReady(state, fetchedArguments, cfCtx, Collections.singletonList(cfCtx.getCfId()), null, null, msg.getCallback()); + } else { + throw new RuntimeException(cfCtx.getSizeExceedsLimitMessage()); + } + } catch (Exception e) { + log.debug("[{}][{}] Failed to initialize CF state", entityId, cfCtx.getCfId(), e); + if (e instanceof CalculatedFieldException cfe) { + throw cfe; + } + throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(entityId).cause(e).build(); + } + } + + @SneakyThrows + private Map fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { + ListenableFuture> argumentsFuture = cfService.fetchAggArguments(ctx, entityId); + // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. + // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. + // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, + // but this will significantly complicate the code. + return argumentsFuture.get(1, TimeUnit.MINUTES); + } + + public void process(EntityCalculatedFieldTelemetryMsg msg) throws CalculatedFieldException { log.trace("[{}] Processing CF telemetry msg: {}", msg.getEntityId(), msg); var proto = msg.getProto(); @@ -462,59 +525,78 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { - return mapToArguments(ctx.getMainEntityArguments(), data); + return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getAggregationInputs(), data); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { - return mapToArguments(ctx.getLinkedAndDynamicArgs(entityId), data); + return mapToArguments(entityId, ctx.getLinkedAndDynamicArgs(entityId), ctx.getAggregationInputs(), data); } - private Map mapToArguments(Map argNames, List data) { - if (argNames.isEmpty()) { - return Collections.emptyMap(); - } + private Map mapToArguments(EntityId originator, Map argNames, Map aggArgNames, List data) { Map arguments = new HashMap<>(); - for (TsKvProto item : data) { - ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); - String argName = argNames.get(key); - if (argName != null) { - arguments.put(argName, new SingleValueArgumentEntry(item)); + if (!aggArgNames.isEmpty()) { + for (Map.Entry entry : aggArgNames.entrySet()) { + for (TsKvProto item : data) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); + if (key.equals(entry.getValue())) { + arguments.put(entry.getKey(), new AggSingleArgumentEntry(originator, item)); + } + } } - key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); - argName = argNames.get(key); - if (argName != null) { - arguments.put(argName, new SingleValueArgumentEntry(item)); + } + if (!argNames.isEmpty()) { + for (TsKvProto item : data) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); + String argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); + argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } } } return arguments; } private Map mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List attrDataList) { - return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), scope, attrDataList); + return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), ctx.getAggregationInputs(), scope, attrDataList); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { var argNames = ctx.getLinkedAndDynamicArgs(entityId); - if (argNames.isEmpty()) { - return Collections.emptyMap(); - } List geofencingArgumentNames = ctx.getLinkedEntityAndCurrentOwnerGeofencingArgumentNames(); - return mapToArguments(entityId, argNames, geofencingArgumentNames, scope, attrDataList); + Map aggregationInputs = ctx.getAggregationInputs(); + return mapToArguments(entityId, argNames, geofencingArgumentNames, aggregationInputs, scope, attrDataList); } - private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, AttributeScopeProto scope, List attrDataList) { + private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, Map aggArgNames, AttributeScopeProto scope, List attrDataList) { Map arguments = new HashMap<>(); - for (AttributeValueProto item : attrDataList) { - ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); - String argName = argNames.get(key); - if (argName == null) { - continue; + if (!argNames.isEmpty()) { + for (AttributeValueProto item : attrDataList) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + String argName = argNames.get(key); + if (argName == null) { + continue; + } + if (geofencingArgNames.contains(argName)) { + arguments.put(argName, new GeofencingArgumentEntry(entityId, item)); + continue; + } + arguments.put(argName, new SingleValueArgumentEntry(item)); } - if (geofencingArgNames.contains(argName)) { - arguments.put(argName, new GeofencingArgumentEntry(entityId, item)); - continue; + } + if (!aggArgNames.isEmpty()) { + for (AttributeValueProto item : attrDataList) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + for (Map.Entry entry : aggArgNames.entrySet()) { + if (key.equals(entry.getValue())) { + arguments.put(entry.getKey(), new AggSingleArgumentEntry(entityId, item)); + } + } } - arguments.put(argName, new SingleValueArgumentEntry(item)); } return arguments; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 4675821a5b..8a717c3cdb 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -25,18 +25,27 @@ import org.thingsboard.server.actors.calculatedField.EntityInitCalculatedFieldMs import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; +import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.msg.CalculatedFieldStatePartitionRestoreMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; @@ -44,10 +53,13 @@ import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldStateService; @@ -64,6 +76,8 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ScheduledFuture; @@ -82,6 +96,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final Map> entityIdCalculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFieldLinks = new HashMap<>(); private final Map> ownerEntities = new HashMap<>(); + private final Map cfTriggers = new HashMap<>(); private ScheduledFuture cfsReevaluationTask; private final CalculatedFieldProcessingService cfExecService; @@ -90,6 +105,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final DeviceService deviceService; private final AssetService assetService; private final CustomerService customerService; + private final RelationService relationService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; private final TenantEntityProfileCache entityProfileCache; @@ -107,6 +123,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware this.deviceService = systemContext.getDeviceService(); this.assetService = systemContext.getAssetService(); this.customerService = systemContext.getCustomerService(); + this.relationService = systemContext.getRelationService(); this.assetProfileCache = systemContext.getAssetProfileCache(); this.deviceProfileCache = systemContext.getDeviceProfileCache(); this.entityProfileCache = new TenantEntityProfileCache(); @@ -129,6 +146,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsReevaluationTask.cancel(true); cfsReevaluationTask = null; } + cfTriggers.clear(); ctx.stop(ctx.getSelf()); } @@ -233,6 +251,20 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware entityProfileCache.add(profileId, entityId); } updateEntityOwner(entityId); + + MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); + + // process aggregation cfs(in any) + List cfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(entityId, profileId); + if (!cfsRelatedToEntity.isEmpty()) { + MultipleTbCallback multiCallback = new MultipleTbCallback(cfsRelatedToEntity.size(), callbackFor2); + cfsRelatedToEntity.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> initRelatedEntity(id, entityId, ctx, cb)); + }); + } else { + callbackFor2.onSuccess(); + } + if (!isMyPartition(entityId, callback)) { return; } @@ -240,11 +272,11 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var profileIdFields = getCalculatedFieldsByEntityId(profileId); var fieldsCount = entityIdFields.size() + profileIdFields.size(); if (fieldsCount > 0) { - MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callbackFor2); entityIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); profileIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); } else { - callback.onSuccess(); + callbackFor2.onSuccess(); } } @@ -254,33 +286,180 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (!isMyPartition(msg.getEntityId(), callback)) { return; } + MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); + + // process aggregation cfs(in any) + List oldCfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getOldProfileId()); + List newCfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getProfileId()); + var fieldsWithRelatedEntityCount = oldCfsRelatedToEntity.size() + newCfsRelatedToEntity.size(); + if (fieldsWithRelatedEntityCount > 0) { + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsWithRelatedEntityCount, callbackFor2); + var entityId = msg.getEntityId(); + oldCfsRelatedToEntity.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> deleteRelatedEntity(id, entityId, cb)); + }); + newCfsRelatedToEntity.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> initRelatedEntity(id, entityId, ctx, cb)); + }); + } else { + callbackFor2.onSuccess(); + } + var oldProfileCfs = getCalculatedFieldsByEntityId(msg.getOldProfileId()); var newProfileCfs = getCalculatedFieldsByEntityId(msg.getProfileId()); var fieldsCount = oldProfileCfs.size() + newProfileCfs.size(); if (fieldsCount > 0) { - MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callbackFor2); var entityId = msg.getEntityId(); oldProfileCfs.forEach(ctx -> deleteCfForEntity(entityId, ctx.getCfId(), multiCallback)); newProfileCfs.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); } else { - callback.onSuccess(); + callbackFor2.onSuccess(); } } else if (msg.isOwnerChanged()) { onEntityOwnerChanged(msg, callback); + } else if (msg.isRelationChanged()) { + onRelationUpdated(msg, callback); } else { callback.onSuccess(); } } private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { - switch (msg.getEntityId().getEntityType()) { - case DEVICE, ASSET -> entityProfileCache.removeEntityId(msg.getEntityId()); - case CUSTOMER -> ownerEntities.remove(msg.getEntityId()); + if (msg.isRelationChanged()) { + onRelationDeleted(msg, callback); + } else { + switch (msg.getEntityId().getEntityType()) { + case DEVICE, ASSET -> entityProfileCache.removeEntityId(msg.getEntityId()); + case CUSTOMER -> ownerEntities.remove(msg.getEntityId()); + } + ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); + + getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getProfileId()).forEach(ctx -> { + applyToTargetCfEntityActors(ctx, callback, (id, cb) -> deleteRelatedEntity(id, msg.getEntityId(), cb)); + }); + if (isMyPartition(msg.getEntityId(), callback)) { + log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); + getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); + } } - ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); - if (isMyPartition(msg.getEntityId(), callback)) { - log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); - getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); + } + + private void onRelationUpdated(ComponentLifecycleMsg msg, TbCallback callback) { + try { + MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); + EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); + + EntityId toId = entityRelation.getTo(); + EntityId fromId = entityRelation.getFrom(); + String relationType = entityRelation.getType(); + EntityId toIdProfile = getProfileId(tenantId, toId); + EntityId fromIdProfile = getProfileId(tenantId, fromId); + + List toIdMatches = new ArrayList<>(); + List cfsByToId = getCalculatedFieldsByEntityId(toId); + List cfsByToProfileId = getCalculatedFieldsByEntityId(toIdProfile); + List cfsByToIdOrItsProfileId = new ArrayList<>(); + cfsByToIdOrItsProfileId.addAll(cfsByToId); + cfsByToIdOrItsProfileId.addAll(cfsByToProfileId); + + cfsByToIdOrItsProfileId.forEach(cf -> { + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(fromIdProfile)) { + toIdMatches.add(cf); + } + }); + + MultipleTbCallback toCfsCallback = new MultipleTbCallback(toIdMatches.size(), callbackForToAndFrom); + toIdMatches.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, toCfsCallback, (entityId, cb) -> initRelatedEntity(entityId, fromId, ctx, cb)); + }); + + List fromIdMatches = new ArrayList<>(); + List cfsByFromId = getCalculatedFieldsByEntityId(fromId); + List cfsByFromProfileId = getCalculatedFieldsByEntityId(fromIdProfile); + List cfsByFromIdOrItsProfileId = new ArrayList<>(); + cfsByFromIdOrItsProfileId.addAll(cfsByFromId); + cfsByFromIdOrItsProfileId.addAll(cfsByFromProfileId); + + cfsByFromIdOrItsProfileId.forEach(cf -> { + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(toIdProfile)) { + fromIdMatches.add(cf); + } + }); + + MultipleTbCallback fromCfsCallback = new MultipleTbCallback(fromIdMatches.size(), callbackForToAndFrom); + fromIdMatches.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, fromCfsCallback, (entityId, cb) -> initRelatedEntity(entityId, toId, ctx, cb)); + }); + + + } catch (Exception e) { + callback.onSuccess(); + } + } + + private void onRelationDeleted(ComponentLifecycleMsg msg, TbCallback callback) { + try { + MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); + EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); + + EntityId toId = entityRelation.getTo(); + EntityId fromId = entityRelation.getFrom(); + String relationType = entityRelation.getType(); + EntityId toIdProfile = getProfileId(tenantId, toId); + EntityId fromIdProfile = getProfileId(tenantId, fromId); + + List toIdMatches = new ArrayList<>(); + List cfsByToId = getCalculatedFieldsByEntityId(toId); + List cfsByToProfileId = getCalculatedFieldsByEntityId(toIdProfile); + List cfsByToIdOrItsProfileId = new ArrayList<>(); + cfsByToIdOrItsProfileId.addAll(cfsByToId); + cfsByToIdOrItsProfileId.addAll(cfsByToProfileId); + + cfsByToIdOrItsProfileId.forEach(cf -> { + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(fromIdProfile)) { + toIdMatches.add(cf); + } + }); + + MultipleTbCallback toCfsCallback = new MultipleTbCallback(toIdMatches.size(), callbackForToAndFrom); + toIdMatches.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, toCfsCallback, (entityId, cb) -> deleteRelatedEntity(entityId, fromId, cb)); + }); + + List fromIdMatches = new ArrayList<>(); + List cfsByFromId = getCalculatedFieldsByEntityId(fromId); + List cfsByFromProfileId = getCalculatedFieldsByEntityId(fromIdProfile); + List cfsByFromIdOrItsProfileId = new ArrayList<>(); + cfsByFromIdOrItsProfileId.addAll(cfsByFromId); + cfsByFromIdOrItsProfileId.addAll(cfsByFromProfileId); + + cfsByFromIdOrItsProfileId.forEach(cf -> { + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(toIdProfile)) { + fromIdMatches.add(cf); + } + }); + + MultipleTbCallback fromCfsCallback = new MultipleTbCallback(fromIdMatches.size(), callbackForToAndFrom); + fromIdMatches.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, fromCfsCallback, (entityId, cb) -> deleteRelatedEntity(entityId, toId, cb)); + }); + + + } catch (Exception e) { + callback.onSuccess(); } } @@ -302,6 +481,9 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } calculatedFields.put(cf.getId(), cfCtx); + if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + } // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); @@ -333,6 +515,9 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(newCfCtx).eventEntity(newCfCtx.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(newCf.getId(), newCfCtx); + if (newCf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(newCf.getId(), aggConfig.buildTrigger()); + } List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new CopyOnWriteArrayList<>(); boolean found = false; @@ -417,6 +602,106 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } else { callback.onSuccess(); } + // process all aggregation cfs (if any); + List aggregationCalculatedFields = filterAggregationCfs(msg); + if (!aggregationCalculatedFields.isEmpty()) { + cfExecService.pushMsgToLinks(msg, aggregationCalculatedFields, callback); + } else { + callback.onSuccess(); + } + } + + private List filterAggregationCfs(CalculatedFieldTelemetryMsg msg) { + EntityId entityId = msg.getEntityId(); + + List aggregationCalculatedFields = cfTriggers.entrySet().stream() + .filter(entry -> aggMatches(entry.getValue(), msg.getProto())) + .map(Entry::getKey) + .map(calculatedFields::get) + .filter(Objects::nonNull) + .toList(); + + List filteredByRelationCfs = new ArrayList<>(); + for (CalculatedFieldCtx cf : aggregationCalculatedFields) { + EntityId cfEntityId = cf.getEntityId(); + if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + RelationPathLevel relation = aggConfig.getSource().getRelation(); + EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) + ? cfEntityId + : getProfileId(tenantId, cfEntityId); + EntityId targetEntity = switch (relation.direction()) { + case FROM -> + relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).getFrom(); + case TO -> + relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0).getTo(); + }; + if (targetEntity != null) { + filteredByRelationCfs.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), targetEntity)); + } + } + } + return filteredByRelationCfs; + } + + private List getCalculatedFieldsRelatedToEntity(EntityId entityId, EntityId profileId) { + List aggCFsUsedProfile = cfTriggers.entrySet().stream() + .filter(entry -> entry.getValue().matchesProfile(profileId)) + .map(Entry::getKey) + .map(calculatedFields::get) + .filter(Objects::nonNull) + .toList(); + + List filteredByRelationCfs = new ArrayList<>(); + for (CalculatedFieldCtx cf : aggCFsUsedProfile) { + CalculatedFieldEntityCtxId calculatedFieldEntityCtxId = filterCfByRelationWithEntity(entityId, cf); + if (calculatedFieldEntityCtxId != null) { + filteredByRelationCfs.add(cf); + } + } + return filteredByRelationCfs; + } + + private CalculatedFieldEntityCtxId filterCfByRelationWithEntity(EntityId entityId, CalculatedFieldCtx cf) { + EntityId cfEntityId = cf.getEntityId(); + if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + RelationPathLevel relation = aggConfig.getSource().getRelation(); + EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) + ? cfEntityId + : getProfileId(tenantId, cfEntityId); + EntityId targetEntity = switch (relation.direction()) { + case FROM -> { + EntityRelation entityRelation = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); + yield entityRelation == null ? null : entityRelation.getFrom(); + } + case TO -> { + EntityRelation entityRelation = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0); + yield entityRelation == null ? null : entityRelation.getTo(); + } + }; + if (targetEntity != null) { + return new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), targetEntity); + } + } + return null; + } + + private boolean aggMatches(CfAggTrigger cfAggTrigger, CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTsDataList().isEmpty()) { + List updatedTelemetry = proto.getTsDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return cfAggTrigger.matchesTimeSeries(updatedTelemetry); + } else if (!proto.getAttrDataList().isEmpty()) { + AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); + List updatedTelemetry = proto.getAttrDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return cfAggTrigger.matchesAttributes(updatedTelemetry, scope); + } else if (!proto.getRemovedTsKeysList().isEmpty()) { + return cfAggTrigger.matchesTimeSeriesKeys(proto.getRemovedTsKeysList()); + } else { + return cfAggTrigger.matchesAttributesKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); + } } public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { @@ -536,6 +821,16 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware getOrCreateActor(entityId).tell(msg); } + private void deleteRelatedEntity(EntityId entityId, EntityId relatedEntityId, TbCallback callback) { + log.debug("Pushing delete related entity msg to specific actor [{}]", relatedEntityId); + getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, relatedEntityId, callback)); + } + + private void initRelatedEntity(EntityId entityId, EntityId relatedEntityId, CalculatedFieldCtx cf, TbCallback callback) { + log.debug("Pushing init related entity msg to specific actor [{}]", relatedEntityId); + getOrCreateActor(entityId).tell(new CalculatedFieldRelatedEntityMsg(tenantId, relatedEntityId, cf, callback)); + } + private void deleteCfForEntity(EntityId entityId, CalculatedFieldId cfId, TbCallback callback) { log.debug("Pushing delete CF msg to specific actor [{}]", entityId); getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, cfId, callback)); @@ -614,6 +909,9 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(cf.getId(), cfCtx); + if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + } // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java new file mode 100644 index 0000000000..d3bdd9cc5c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +@Data +public class CalculatedFieldRelatedEntityMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldCtx calculatedField; + private final TbCallback callback; + + public CalculatedFieldRelatedEntityMsg(TenantId tenantId, + EntityId entityId, + CalculatedFieldCtx calculatedField, + TbCallback callback) { + this.tenantId = tenantId; + this.entityId = entityId; + this.calculatedField = calculatedField; + this.callback = callback; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_RELATED_ENTITY_MSG; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index d17148a502..304b747f26 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -26,7 +26,10 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; @@ -36,6 +39,8 @@ import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.relation.RelationService; @@ -44,7 +49,9 @@ import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -98,6 +105,7 @@ public abstract class AbstractCalculatedFieldProcessingService { } yield futures; } + case LATEST_VALUES_AGGREGATION -> fetchAggregationArgumentFutures(ctx, entityId); }; return Futures.whenAllComplete(argFutures.values()) .call(() -> resolveArgumentFutures(argFutures), @@ -114,6 +122,19 @@ public abstract class AbstractCalculatedFieldProcessingService { return resolveOwnerArgument(tenantId, entityId); } + private List resolveRelatedEntities(TenantId tenantId, EntityId entityId, AggSource aggSource) { + RelationPathLevel relation = aggSource.getRelation(); + return switch (relation.direction()) { + case FROM -> aggSource.getEntityProfiles().stream() + .map(profile -> relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), profile)) + .flatMap(Collection::stream) + .map(EntityRelation::getTo) + .toList(); + case TO -> + aggSource.getEntityProfiles().stream().map(profile -> relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), profile).getFrom()).toList(); + }; + } + protected Map resolveArgumentFutures(Map> argFutures) { return argFutures.entrySet().stream() .collect(Collectors.toMap( @@ -176,6 +197,50 @@ public abstract class AbstractCalculatedFieldProcessingService { return ownerService.getOwner(tenantId, entityId); } + private Map> fetchAggregationArgumentFutures(CalculatedFieldCtx ctx, EntityId entityId) { + LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + + List entityIds = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getSource()); + + Map> futures = new HashMap<>(); + aggConfig.getInputs().forEach((key, refKey) -> { + Argument argument = new Argument(); + argument.setRefEntityKey(refKey); + futures.put(key, fetchAggArgumentEntry(ctx.getTenantId(), entityIds, argument, System.currentTimeMillis())); + }); + return futures; + } + + public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) { + List>> futures = aggEntities.stream() + .map(entityId -> fetchSingleAggArgumentEntry(tenantId, entityId, argument, startTs)) + .toList(); + + ListenableFuture>> allFutures = Futures.allAsList(futures); + + return Futures.transform(allFutures, + entries -> ArgumentEntry.createAggArgument( + entries.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) + ), + MoreExecutors.directExecutor()); + } + + protected ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { + CalculatedFieldConfiguration configuration = ctx.getCalculatedField().getConfiguration(); + LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) configuration; + Map> futures = new HashMap<>(); + aggConfig.getInputs().forEach((key, refKey) -> { + Argument argument = new Argument(); + argument.setRefEntityKey(refKey); + + ListenableFuture argumentEntryListenableFuture = fetchAggArgumentEntry(ctx.getTenantId(), List.of(entityId), argument, System.currentTimeMillis()); + futures.put(key, argumentEntryListenableFuture); + }); + return Futures.whenAllComplete(futures.values()) + .call(() -> resolveArgumentFutures(futures), + MoreExecutors.directExecutor()); + } + private ListenableFuture fetchGeofencingKvEntry(TenantId tenantId, List geofencingEntities, Argument argument) { if (argument.getRefEntityKey().getType() != ArgumentType.ATTRIBUTE) { throw new IllegalStateException("Unsupported argument key type: " + argument.getRefEntityKey().getType()); @@ -244,6 +309,38 @@ public abstract class AbstractCalculatedFieldProcessingService { }, calculatedFieldCallbackExecutor)); } + protected ListenableFuture> fetchSingleAggArgumentEntry(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { + return switch (argument.getRefEntityKey().getType()) { + case TS_ROLLING -> throw new IllegalStateException("TS_ROLLING is not supported for aggregation"); + case ATTRIBUTE -> fetchAttributeAggEntry(tenantId, entityId, argument, startTs); + case TS_LATEST -> fetchTsLatestAggEntry(tenantId, entityId, argument, startTs); + }; + } + + private ListenableFuture> fetchAttributeAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultLastUpdateTs) { + log.trace("[{}][{}] Fetching attribute for key {}", tenantId, entityId, argument.getRefEntityKey()); + var attributeOptFuture = attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()); + return Futures.transform(attributeOptFuture, attrOpt -> { + log.debug("[{}][{}] Fetched attribute for key {}: {}", tenantId, entityId, argument.getRefEntityKey(), attrOpt); + AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, 0L)); + AggSingleArgumentEntry entry = new AggSingleArgumentEntry(entityId, attributeKvEntry); + return Map.entry(entityId, entry); + }, calculatedFieldCallbackExecutor); + } + + protected ListenableFuture> fetchTsLatestAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultTs) { + String key = argument.getRefEntityKey().getKey(); + log.trace("[{}][{}] Fetching latest timeseries {}", tenantId, entityId, key); + return Futures.transform( + timeseriesService.findLatest(tenantId, entityId, key), + result -> { + log.debug("[{}][{}] Fetched latest timeseries {}: {}", tenantId, entityId, key, result); + Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))); + AggSingleArgumentEntry entry = new AggSingleArgumentEntry(entityId, tsKvEntry.get()); + return Map.entry(entityId, entry); + }, calculatedFieldCallbackExecutor); + } + private ReadTsKvQuery buildTsRollingQuery(TenantId tenantId, Argument argument, long startTs, long endTs) { long maxDataPoints = apiLimitService.getLimit( tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index cc77913f4b..e32ca42f9c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -38,6 +39,8 @@ public interface CalculatedFieldCache { List getCalculatedFieldCtxsByEntityId(EntityId entityId); + List getCalculatedFieldCtxsByTrigger(EntityId profileId, Predicate cfAggFilter); + boolean hasCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter); void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index a9139572b8..4b3e994f23 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -33,6 +33,8 @@ public interface CalculatedFieldProcessingService { ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId); + ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId); + Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId); Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 36210d7302..ae8238fb8a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; @@ -41,6 +43,8 @@ import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -48,6 +52,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Predicate; +import java.util.stream.Collectors; @Service @Slf4j @@ -68,6 +73,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + private final ConcurrentMap cfTriggers = new ConcurrentHashMap<>(); private final ConcurrentMap> ownerEntities = new ConcurrentHashMap<>(); @@ -81,6 +87,9 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { cfs.forEach(cf -> { if (cf != null) { calculatedFields.putIfAbsent(cf.getId(), cf); + if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + } } }); calculatedFields.values().forEach(cf -> { @@ -146,6 +155,16 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { .toList(); } + @Override + public List getCalculatedFieldCtxsByTrigger(EntityId profileId, Predicate cfAggFilter) { + return cfTriggers.entrySet().stream() + .filter(entry -> entry.getValue().matches(profileId, cfAggFilter)) + .map(Map.Entry::getKey) + .map(this::getCalculatedFieldCtx) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + @Override public boolean hasCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter) { List entityCfs = getCalculatedFieldCtxsByEntityId(entityId); @@ -155,6 +174,10 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } } + return hasCalculatedFieldsByProfile(tenantId, entityId, filter); + } + + public boolean hasCalculatedFieldsByProfile(TenantId tenantId, EntityId entityId, Predicate filter) { EntityId profileId = getProfileId(tenantId, entityId); if (profileId != null) { List profileCfs = getCalculatedFieldCtxsByEntityId(profileId); @@ -183,6 +206,9 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + if (configuration instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(calculatedField.getId(), aggConfig.buildTrigger()); + } calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); configuration.getReferencedEntities().stream() @@ -214,6 +240,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); entityIdCalculatedFieldLinks.forEach((entityId, calculatedFieldLinks) -> calculatedFieldLinks.removeIf(link -> link.getCalculatedFieldId().equals(calculatedFieldId))); log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); + cfTriggers.remove(calculatedFieldId); + log.debug("[{}] evict calculated field from cached triggers: {}", calculatedFieldId, oldCalculatedField); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index 9b2964a736..b7bd4d87fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.service.cf; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; @@ -54,6 +56,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; @@ -87,6 +90,11 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF return super.fetchArguments(ctx, entityId, System.currentTimeMillis()); } + @Override + public ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { + return super.fetchAggArguments(ctx, entityId, System.currentTimeMillis()); + } + @Override public Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId) { // only scheduledSupported CF instances supports dynamic arguments scheduled updates diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index a3e049d1e2..a75df1fb40 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -35,7 +37,10 @@ import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -71,6 +76,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS private final CalculatedFieldCache calculatedFieldCache; private final TbClusterService clusterService; + private final RelationService relationService; @Override public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { @@ -81,6 +87,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matches(entries), cf -> cf.linkMatches(entityId, entries), cf -> cf.dynamicSourceMatches(request.getEntries()), + cfTrigger -> cfTrigger.matchesTimeSeries(entries), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -99,6 +106,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matches(entries, scope), cf -> cf.linkMatches(entityId, entries, scope), cf -> cf.dynamicSourceMatches(request.getEntries(), request.getScope()), + cfTrigger -> cfTrigger.matchesAttributes(entries, scope), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -116,6 +124,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matchesKeys(result, scope), cf -> cf.linkMatchesAttrKeys(entityId, result, scope), cf -> cf.matchesDynamicSourceKeys(result, request.getScope()), + cfTrigger -> cfTrigger.matchesAttributesKeys(result, scope), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -127,6 +136,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matchesKeys(result), cf -> cf.linkMatchesTsKeys(entityId, result), cf -> cf.matchesDynamicSourceKeys(result), + cfTrigger -> cfTrigger.matchesTimeSeriesKeys(result), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -134,11 +144,12 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS Predicate mainEntityFilter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter, + Predicate cfAggKeysFilter, Supplier msg, FutureCallback callback) { if (EntityType.TENANT.equals(entityId.getEntityType())) { tenantId = (TenantId) entityId; } - boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter, dynamicSourceFilter); + boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter, dynamicSourceFilter, cfAggKeysFilter); if (send) { ToCalculatedFieldMsg calculatedFieldMsg = msg.get(); clusterService.pushMsgToCalculatedFields(tenantId, entityId, calculatedFieldMsg, wrap(callback)); @@ -149,7 +160,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } } - private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter) { + private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter, Predicate cfAggKeysFilter) { if (!CalculatedField.SUPPORTED_REFERENCED_ENTITIES.contains(entityId.getEntityType())) { return false; } @@ -176,6 +187,26 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } } + List cfCtxs = calculatedFieldCache.getCalculatedFieldCtxsByTrigger(calculatedFieldCache.getProfileId(tenantId, entityId), cfAggKeysFilter); + for (CalculatedFieldCtx cfCtx : cfCtxs) { + EntityId cfEntityId = cfCtx.getEntityId(); + if (cfCtx.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + RelationPathLevel relation = aggConfig.getSource().getRelation(); + EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) + ? cfEntityId + : calculatedFieldCache.getProfileId(tenantId, cfEntityId); + EntityRelation entityRelation = switch (relation.direction()) { + case FROM -> + relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); + case TO -> + relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0); + }; + if (entityRelation != null) { + return true; + } + } + } + return false; } @@ -266,6 +297,10 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS return telemetryMsg; } + private boolean isProfileEntity(EntityType entityType) { + return EntityType.DEVICE_PROFILE.equals(entityType) || EntityType.ASSET_PROFILE.equals(entityType); + } + private static TbQueueCallback wrap(FutureCallback callback) { if (callback != null) { return new FutureCallbackWrapper(callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 2d43883131..f22b10a9c1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -22,6 +22,8 @@ import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import java.util.List; @@ -35,7 +37,9 @@ import java.util.Map; @JsonSubTypes({ @JsonSubTypes.Type(value = SingleValueArgumentEntry.class, name = "SINGLE_VALUE"), @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING"), - @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING") + @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING"), + @JsonSubTypes.Type(value = AggArgumentEntry.class, name = "AGGREGATE_LATEST"), + @JsonSubTypes.Type(value = AggSingleArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") }) public interface ArgumentEntry { @@ -66,4 +70,8 @@ public interface ArgumentEntry { return new GeofencingArgumentEntry(entityIdkvEntryMap); } + static ArgumentEntry createAggArgument(Map entityIdkvEntryMap) { + return new AggArgumentEntry(entityIdkvEntryMap, false); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java index 876bfa2a3f..30b92a78d3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java @@ -16,5 +16,5 @@ package org.thingsboard.server.service.cf.ctx.state; public enum ArgumentEntryType { - SINGLE_VALUE, TS_ROLLING, GEOFENCING + SINGLE_VALUE, TS_ROLLING, GEOFENCING, AGGREGATE_LATEST, AGGREGATE_LATEST_SINGLE } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index e442964280..f75711a107 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -123,7 +123,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, protected void validateNewEntry(String key, ArgumentEntry newEntry) {} - private void updateLastUpdateTimestamp(ArgumentEntry entry) { + protected void updateLastUpdateTimestamp(ArgumentEntry entry) { long newTs = this.latestTimestamp; if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { newTs = singleValueArgumentEntry.getTs(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index e08abb8b50..963ec3f06f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -42,6 +42,8 @@ import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.ScheduledUpdateSupportedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -81,6 +83,8 @@ public class CalculatedFieldCtx { private final Map mainEntityArguments; private final Map> linkedEntityArguments; private final Map dynamicEntityArguments; + private final List aggInputs; + private final Map aggregationInputs; private final List argNames; private Output output; private String expression; @@ -122,6 +126,8 @@ public class CalculatedFieldCtx { this.argNames = new ArrayList<>(); this.mainEntityGeofencingArgumentNames = new ArrayList<>(); this.linkedEntityAndCurrentOwnerGeofencingArgumentNames = new ArrayList<>(); + this.aggInputs = new ArrayList<>(); + this.aggregationInputs = new HashMap<>(); this.output = calculatedField.getConfiguration().getOutput(); if (calculatedField.getConfiguration() instanceof ArgumentsBasedCalculatedFieldConfiguration argBasedConfig) { this.arguments.putAll(argBasedConfig.getArguments()); @@ -165,6 +171,12 @@ public class CalculatedFieldCtx { this.scheduledUpdateIntervalMillis = scheduledConfig.isScheduledUpdateEnabled() ? TimeUnit.SECONDS.toMillis(scheduledConfig.getScheduledUpdateInterval()) : -1L; } this.requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); + if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + aggInputs.addAll(aggConfig.getInputs().values()); + aggregationInputs.putAll(aggConfig.getInputs()); + this.argNames.addAll(aggConfig.getInputs().keySet()); + this.scheduledUpdateIntervalMillis = aggConfig.getDeduplicationIntervalMillis(); + } this.systemContext = systemContext; this.tbelInvokeService = systemContext.getTbelInvokeService(); this.relationService = systemContext.getRelationService(); @@ -199,6 +211,19 @@ public class CalculatedFieldCtx { }); initialized = true; } + case LATEST_VALUES_AGGREGATION -> { + LatestValuesAggregationCalculatedFieldConfiguration configuration = (LatestValuesAggregationCalculatedFieldConfiguration) calculatedField.getConfiguration(); + configuration.getMetrics().forEach((key, metric) -> { + if (metric.getInput() instanceof AggFunctionInput functionInput) { + initTbelExpression(functionInput.getFunction()); + } + String filter = metric.getFilter(); + if (filter != null && !filter.isEmpty()) { + initTbelExpression(filter); + } + }); + initialized = true; + } } } @@ -242,6 +267,24 @@ public class CalculatedFieldCtx { return expression.executeScriptAsync(args.toArray()); } + public ListenableFuture evaluateTbelExpression(String expression, Map entries, long latestTimestamp) { + Map arguments = new LinkedHashMap<>(); + List args = new ArrayList<>(argNames.size() + 1); + args.add(new Object()); // first element is a ctx, but we will set it later; + for (String argName : argNames) { + var arg = entries.get(argName).toTbelCfArg(); + arguments.put(argName, arg); + if (arg instanceof TbelCfSingleValueArg svArg) { + args.add(svArg.getValue()); + } else { + args.add(arg); + } + } + args.set(0, new TbelCfCtx(arguments, latestTimestamp)); + + return tbelExpressions.get(expression).executeScriptAsync(args.toArray()); + } + public ScheduledFuture scheduleReevaluation(long delayMs, TbActorRef actorCtx) { log.debug("[{}] Scheduling CF reevaluation in {} ms", cfId, delayMs); // TODO: use single lazy-loaded instance of CalculatedFieldReevaluateMsg @@ -451,6 +494,25 @@ public class CalculatedFieldCtx { } } + public boolean aggMatches(CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTsDataList().isEmpty()) { + List updatedTelemetry = proto.getTsDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return matchesAggTimeSeries(updatedTelemetry); + } else if (!proto.getAttrDataList().isEmpty()) { + AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); + List updatedTelemetry = proto.getAttrDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return matchesAggAttributes(updatedTelemetry, scope); + } else if (!proto.getRemovedTsKeysList().isEmpty()) { + return matchesAggKeys(proto.getRemovedTsKeysList()); + } else { + return matchesAggAttributesKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); + } + } + public boolean linkMatches(EntityId entityId, CalculatedFieldTelemetryMsgProto proto) { if (!proto.getTsDataList().isEmpty()) { List updatedTelemetry = proto.getTsDataList().stream() @@ -482,6 +544,67 @@ public class CalculatedFieldCtx { return argNames; } + public boolean matchesAggKeys(List values) { + if (aggInputs.isEmpty() || values.isEmpty()) { + return false; + } + + for (String key : values) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); + if (aggInputs.contains(latestKey)) { + return true; + } + } + + return false; + } + + public boolean matchesAggTimeSeries(List values) { + if (aggInputs.isEmpty() || values.isEmpty()) { + return false; + } + + for (TsKvEntry tsKvEntry : values) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKvEntry.getKey(), ArgumentType.TS_LATEST, null); + if (aggInputs.contains(latestKey)) { + return true; + } + } + + return false; + } + + public boolean matchesAggAttributesKeys(List keys, AttributeScope scope) { + if (keys == null || keys.isEmpty()) { + return false; + } + + for (String key : keys) { + ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); + if (aggInputs.contains(attrKey)) { + return true; + } + } + + return false; + } + + public boolean matchesAggAttributes(List keys, AttributeScope scope) { + if (keys == null || keys.isEmpty()) { + return false; + } + + for (AttributeKvEntry attributeKvEntry : keys) { + ReferencedEntityKey attrKey = new ReferencedEntityKey(attributeKvEntry.getKey(), ArgumentType.ATTRIBUTE, scope); + if (aggInputs.contains(attrKey)) { + return true; + } + } + + return false; + } + + public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); } @@ -499,6 +622,11 @@ public class CalculatedFieldCtx { if (scheduledUpdateIntervalMillis != other.scheduledUpdateIntervalMillis) { return true; } + if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig + && other.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig + && !thisConfig.getMetrics().equals(otherConfig.getMetrics())) { + return true; + } return false; } @@ -517,6 +645,9 @@ public class CalculatedFieldCtx { if (hasGeofencingZoneGroupConfigurationChanges(other)) { return true; } + if (hasLatestValuesAggregationConfigurationChanges(other)) { + return true; + } return false; } @@ -528,6 +659,15 @@ public class CalculatedFieldCtx { return false; } + private boolean hasLatestValuesAggregationConfigurationChanges(CalculatedFieldCtx other) { + if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig + && other.calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig) { + return !thisConfig.getInputs().equals(otherConfig.getInputs()) || !thisConfig.getSource().equals(otherConfig.getSource()); + } + return false; + } + + public boolean hasRelationQueryDynamicArguments() { return relationQueryDynamicArguments && scheduledUpdateIntervalMillis != -1; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index cc7188e7a5..bd40ab7a05 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -32,6 +32,7 @@ import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculat import java.io.Closeable; import java.util.Map; +import java.util.concurrent.ExecutionException; import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArgumentProto; @@ -53,6 +54,8 @@ public interface CalculatedFieldState extends Closeable { long getLatestTimestamp(); + CalculatedFieldCtx getCtx(); + void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx); void init(); @@ -61,7 +64,7 @@ public interface CalculatedFieldState extends Closeable { void reset(); - ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx); + ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception; @JsonIgnore boolean isReady(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 5c1ed32e1d..288b486e83 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -37,11 +37,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; @AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { - private long ts; - private BasicKvEntry kvEntryValue; - private Long version; + protected long ts; + protected BasicKvEntry kvEntryValue; + protected Long version; - private boolean forceResetPrevious; + protected boolean forceResetPrevious; public static final Long DEFAULT_VERSION = -1L; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java new file mode 100644 index 0000000000..138053793f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; + +import java.util.Map; + +@Data +@AllArgsConstructor +public class AggArgumentEntry implements ArgumentEntry { + + private final Map aggInputs; + + private boolean forceResetPrevious; + + @Override + public ArgumentEntryType getType() { + return ArgumentEntryType.AGGREGATE_LATEST; + } + + @Override + public Object getValue() { + return aggInputs; + } + + @Override + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof AggArgumentEntry aggArgumentEntry) { + aggInputs.putAll(aggArgumentEntry.aggInputs); + return true; + } else if (entry instanceof AggSingleArgumentEntry aggSingleArgumentEntry) { + if (aggSingleArgumentEntry.isDeleted()) { + aggInputs.remove(aggSingleArgumentEntry.getEntityId()); + } else { + aggInputs.put(aggSingleArgumentEntry.getEntityId(), aggSingleArgumentEntry); + } + return true; + } else { + throw new IllegalArgumentException("Unsupported argument entry type for aggregation argument entry: " + entry.getType()); + } + } + + @Override + public boolean isEmpty() { + return aggInputs.isEmpty(); + } + + @Override + public TbelCfArg toTbelCfArg() { + return null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java new file mode 100644 index 0000000000..6b81d5380c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java @@ -0,0 +1,83 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class AggSingleArgumentEntry extends SingleValueArgumentEntry { + + private EntityId entityId; + private boolean deleted; + + public AggSingleArgumentEntry(EntityId entityId, TsKvProto entry) { + super(entry); + this.entityId = entityId; + } + + public AggSingleArgumentEntry(EntityId entityId, AttributeValueProto entry) { + super(entry); + this.entityId = entityId; + } + + public AggSingleArgumentEntry(EntityId entityId, KvEntry entry) { + super(entry); + this.entityId = entityId; + } + + public AggSingleArgumentEntry(EntityId entityId, long ts, BasicKvEntry kvEntryValue, Long version) { + super(ts, kvEntryValue, version); + this.entityId = entityId; + } + + @Override + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof AggSingleArgumentEntry singleValueEntry) { + if (singleValueEntry.getTs() <= ts) { + return false; + } + + Long newVersion = singleValueEntry.getVersion(); + if (newVersion == null || this.version == null || newVersion > this.version) { + this.ts = singleValueEntry.getTs(); + this.version = newVersion; + this.kvEntryValue = singleValueEntry.getKvEntryValue(); + this.entityId = singleValueEntry.getEntityId(); + return true; + } + } else { + throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); + } + return false; + } + + @Override + public ArgumentEntryType getType() { + return ArgumentEntryType.AGGREGATE_LATEST_SINGLE; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java new file mode 100644 index 0000000000..d25cde020c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java @@ -0,0 +1,161 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.actors.TbActorRef; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.BaseCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.aggregation.function.AggEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.function.AggFunctionFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +@Slf4j +@Data +public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedFieldState { + + private long lastArgsRefreshTs = -1; + private long lastMetricsEvalTs = -1; + private long deduplicationInterval = -1; + private Map metrics; + + private final Map> inputs = new HashMap<>(); + + public LatestValuesAggregationCalculatedFieldState(EntityId entityId) { + super(entityId); + } + + @Override + public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) { + super.setCtx(ctx, actorCtx); + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + metrics = configuration.getMetrics(); + deduplicationInterval = configuration.getDeduplicationIntervalMillis(); + } + + @Override + public void reset() { // must reset everything dependent on arguments + super.reset(); + lastArgsRefreshTs = -1; + lastMetricsEvalTs = -1; + metrics = null; + } + + @Override + public void init() { + super.init(); +// long scheduledUpdateIntervalMillis = ctx.getScheduledUpdateIntervalMillis(); +// ctx.scheduleReevaluation(scheduledUpdateIntervalMillis, actorCtx); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.LATEST_VALUES_AGGREGATION; + } + + @Override + public Map update(Map argumentValues, CalculatedFieldCtx ctx) { + Map updatedArguments = super.update(argumentValues, ctx); + lastArgsRefreshTs = System.currentTimeMillis(); + for (Map.Entry argEntry : arguments.entrySet()) { + String key = argEntry.getKey(); + AggArgumentEntry aggArgumentEntry = (AggArgumentEntry) argEntry.getValue(); + Map aggInputs = aggArgumentEntry.getAggInputs(); + aggInputs.forEach((entityId, argumentEntry) -> { + inputs.computeIfAbsent(entityId, k -> new HashMap<>()).put(key, argumentEntry); + }); + } + return updatedArguments; + } + + @Override + public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception { + boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - deduplicationInterval; + boolean argsUpdatedDuringInterval = lastArgsRefreshTs > lastMetricsEvalTs; + if (intervalPassed && argsUpdatedDuringInterval) { + ObjectNode aggResult = JacksonUtil.newObjectNode(); + for (Entry entry : metrics.entrySet()) { + String metricKey = entry.getKey(); + AggMetric metric = entry.getValue(); + + AggEntry aggMetric = AggFunctionFactory.createAggFunction(metric.getFunction()); + + for (Map entityInputs : inputs.values()) { + if (applyAggregation(metric.getFilter(), entityInputs)) { + Object arg = resolveAggregationInput(metric.getInput(), entityInputs); + if (arg != null) { + aggMetric.update(arg); + } + } + } + + aggMetric.result().ifPresent(result -> { + aggResult.set(metricKey, JacksonUtil.valueToTree(result)); + }); + } + Output output = ctx.getOutput(); + lastMetricsEvalTs = System.currentTimeMillis(); + ctx.scheduleReevaluation(deduplicationInterval, actorCtx); + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .type(output.getType()) + .scope(output.getScope()) + .result(aggResult) + .build()); + } else { + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .result(null) + .build()); + } + } + + private boolean applyAggregation(String filter, Map entityInputs) throws Exception { + if (filter == null || filter.isEmpty()) { + return true; + } else { + Object filterResult = ctx.evaluateTbelExpression(filter, entityInputs, getLatestTimestamp()).get(); + return filterResult instanceof Boolean booleanResult && booleanResult; + } + } + + private Object resolveAggregationInput(AggInput aggInput, Map entityInputs) throws Exception { + if (aggInput instanceof AggFunctionInput functionInput) { + return ctx.evaluateTbelExpression(functionInput.getFunction(), entityInputs, getLatestTimestamp()).get(); + } else { + String inputKey = ((AggKeyInput) aggInput).getKey(); + return entityInputs.get(inputKey).getValue(); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json new file mode 100644 index 0000000000..d402f23c6e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json @@ -0,0 +1,65 @@ +{ + "type": "LATEST_VALUES_AGGREGATION", + "name": "Occupied spaces", + "debugSettings": { + "failuresEnabled": true, + "allEnabled": true, + "allEnabledUntil": 1769907492297 + }, + "entityId": { + "entityType": "ASSET", + "id": "cc830710-a4cf-11f0-87cb-2d6683c4fccf" + }, + "configuration": { + "type": "LATEST_VALUES_AGGREGATION", + "source": { + "relation": { + "direction": "FROM", + "relationType": "Contains" + }, + "entityProfiles": [ + { + "entityType": "DEVICE_PROFILE", + "id": "d7a05580-a4cf-11f0-87cb-2d6683c4fccf" + } + ] + }, + "inputs": { + "oc": { + "key": "occupied", + "type": "TS_LATEST" + } + }, + "deduplicationIntervalMillis": 10000, + "metrics": { + "totalSpaces": { + "function": "COUNT", + "input": { + "type": "function", + "function" : "return 1;" + } + }, + "occupiedSpaces": { + "function": "COUNT", + "filter": "return oc == true", + "input": { + "type": "key", + "key" : "oc" + } + }, + "freeSpaces": { + "function": "COUNT", + "filter": "return oc == false", + "input": { + "type": "key", + "key" : "oc" + } + } + }, + "output": { + "type": "TIME_SERIES", + "decimals": 2 + }, + "useLatestTsFromInputs": "true" + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java new file mode 100644 index 0000000000..4239e94fec --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.util.Optional; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = AvgAggEntry.class, name = "AVG"), + @JsonSubTypes.Type(value = CountAggEntry.class, name = "COUNT"), + @JsonSubTypes.Type(value = CountUniqueAggEntry.class, name = "COUNT_UNIQUE"), + @JsonSubTypes.Type(value = MaxAggEntry.class, name = "MAX"), + @JsonSubTypes.Type(value = MinAggEntry.class, name = "MIN"), + @JsonSubTypes.Type(value = SumAggEntry.class, name = "SUM") +}) +public interface AggEntry { + + AggFunction getType(); + + void update(Object value); + + Optional result(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java new file mode 100644 index 0000000000..5ccc355b1f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +public class AggFunctionFactory { + + public static AggEntry createAggFunction(AggFunction function) { + return switch (function) { + case MIN -> new MinAggEntry(); + case MAX -> new MaxAggEntry(); + case SUM -> new SumAggEntry(); + case AVG -> new AvgAggEntry(); + case COUNT -> new CountAggEntry(); + case COUNT_UNIQUE -> new CountUniqueAggEntry(); + }; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java new file mode 100644 index 0000000000..ad1f2ee8a8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.math.BigDecimal; +import java.math.RoundingMode; + +public class AvgAggEntry extends BaseAggEntry { + + private BigDecimal sum = BigDecimal.ZERO; + private long count = 0L; + + @Override + protected void doUpdate(double value) { + if (value != 0.0) { + sum = sum.add(BigDecimal.valueOf(value)); + } + this.count++; + } + + @Override + protected double prepareResult() { + return sum.divide(BigDecimal.valueOf(count), 2, RoundingMode.HALF_UP).doubleValue(); + } + + @Override + public AggFunction getType() { + return AggFunction.AVG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java new file mode 100644 index 0000000000..0c12bd13c0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import java.util.Optional; + +public abstract class BaseAggEntry implements AggEntry { + + private boolean hasResult = false; + + @Override + public void update(Object value) { + doUpdate(extractDoubleValue(value)); + hasResult = true; + } + + @Override + public Optional result() { + if (hasResult) { + hasResult = false; + return Optional.of(prepareResult()); + } else { + return Optional.empty(); + } + } + + protected abstract void doUpdate(double value); + + protected abstract double prepareResult(); + + protected double extractDoubleValue(Object value) { + try { + return Double.parseDouble(value.toString()); + } catch (Exception e) { + throw new NumberFormatException("Cannot parse value " + value.toString()); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java new file mode 100644 index 0000000000..469048d62d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.util.Optional; + +public class CountAggEntry implements AggEntry { + + private long count = 0L; + + @Override + public void update(Object value) { + count++; + } + + @Override + public Optional result() { + return Optional.of(count); + } + + @Override + public AggFunction getType() { + return AggFunction.COUNT; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java new file mode 100644 index 0000000000..b8b0b92470 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +public class CountUniqueAggEntry implements AggEntry { + + private Set items; + + @Override + public void update(Object value) { + if (value != null) { + items.add(JacksonUtil.toString(value)); + } + } + + @Override + public Optional result() { + return Optional.of(items.size()); + } + + @Override + public AggFunction getType() { + return AggFunction.COUNT_UNIQUE; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java new file mode 100644 index 0000000000..6e4235b72f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +public class MaxAggEntry extends BaseAggEntry { + + private double max = -Double.MAX_VALUE; + + @Override + protected void doUpdate(double value) { + if (value > max) { + max = value; + } + } + + @Override + protected double prepareResult() { + return max; + } + + @Override + public AggFunction getType() { + return AggFunction.MAX; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java new file mode 100644 index 0000000000..eeacc3a7d9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +public class MinAggEntry extends BaseAggEntry { + + private double min = Double.MAX_VALUE; + + @Override + protected void doUpdate(double value) { + if (value < min) { + min = value; + } + } + + @Override + protected double prepareResult() { + return min; + } + + @Override + public AggFunction getType() { + return AggFunction.MIN; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java new file mode 100644 index 0000000000..b90817d784 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.math.BigDecimal; + +public class SumAggEntry extends BaseAggEntry { + + private BigDecimal sum = BigDecimal.ZERO; + + @Override + protected void doUpdate(double value) { + if (value != 0.0) { + sum = sum.add(BigDecimal.valueOf(value)); + } + } + + @Override + protected double prepareResult() { + return sum.doubleValue(); + } + + @Override + public AggFunction getType() { + return AggFunction.SUM; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index b9fd38f1e2..f762cf5be5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; @@ -61,6 +62,7 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeSynchronizationManager; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; +import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.gen.transport.TransportProtos.EntityActionEventProto; @@ -271,6 +273,15 @@ public class EntityStateSourcingListener { } } + @TransactionalEventListener(fallbackExecution = true) + public void handleEvent(RelationActionEvent relationEvent) { + if (relationEvent.getActionType() == ActionType.RELATION_ADD_OR_UPDATE) { + tbClusterService.onRelationUpdated(relationEvent.getTenantId(), relationEvent.getRelation(), TbQueueCallback.EMPTY); + } else if (relationEvent.getActionType() == ActionType.RELATION_DELETED) { + tbClusterService.onRelationDeleted(relationEvent.getTenantId(), relationEvent.getRelation(), TbQueueCallback.EMPTY); + } + } + private void onTenantUpdate(Tenant tenant, ComponentLifecycleEvent lifecycleEvent) { tbClusterService.onTenantChange(tenant, null); tbClusterService.broadcastEntityStateChangeEvent(tenant.getId(), tenant.getId(), lifecycleEvent); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index f1494658a8..22faeaf61b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -56,6 +56,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; @@ -369,6 +370,17 @@ public class DefaultTbClusterService implements TbClusterService { broadcast(new ComponentLifecycleMsg(tenantId, entityId, state)); } + @Override + public void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, EntityId profileId, ComponentLifecycleEvent state) { + log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); + broadcast(ComponentLifecycleMsg.builder() + .tenantId(tenantId) + .entityId(entityId) + .profileId(profileId) + .event(state) + .build()); + } + @Override public void onDeviceProfileChange(DeviceProfile deviceProfile, DeviceProfile oldDeviceProfile, TbQueueCallback callback) { boolean isFirmwareChanged = false; @@ -420,13 +432,13 @@ public class DefaultTbClusterService implements TbClusterService { gatewayNotificationsService.onDeviceDeleted(device); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); - broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); + broadcastEntityStateChangeEvent(tenantId, deviceId, device.getDeviceProfileId(), ComponentLifecycleEvent.DELETED); } @Override public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { AssetId assetId = asset.getId(); - broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); + broadcastEntityStateChangeEvent(tenantId, assetId, asset.getAssetProfileId(), ComponentLifecycleEvent.DELETED); } @Override @@ -723,6 +735,30 @@ public class DefaultTbClusterService implements TbClusterService { broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED); } + @Override + public void onRelationUpdated(TenantId tenantId, EntityRelation entityRelation, TbQueueCallback callback) { + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(tenantId) + .entityId(entityRelation.getFrom()) + .relationChanged(true) + .event(ComponentLifecycleEvent.UPDATED) + .info(JacksonUtil.valueToTree(entityRelation)) + .build(); + broadcast(msg); + } + + @Override + public void onRelationDeleted(TenantId tenantId, EntityRelation entityRelation, TbQueueCallback callback) { + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(tenantId) + .entityId(entityRelation.getFrom()) + .relationChanged(true) + .event(ComponentLifecycleEvent.DELETED) + .info(JacksonUtil.valueToTree(entityRelation)) + .build(); + broadcast(msg); + } + @Override public void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId originatorEdgeId) { if (!edgesEnabled) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 7382ef1c4d..761b69024f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -184,24 +184,34 @@ public abstract class AbstractConsumerService new ScriptCalculatedFieldState(entityId); case GEOFENCING -> new GeofencingCalculatedFieldState(entityId); case ALARM -> new AlarmCalculatedFieldState(entityId); + case LATEST_VALUES_AGGREGATION -> new LatestValuesAggregationCalculatedFieldState(entityId); }; } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 4319f72448..d4ee7bb2e3 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.util.KvProtoUtil; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.AggSingleArgumentEntryProto; import org.thingsboard.server.gen.transport.TransportProtos.AlarmRuleStateProto; import org.thingsboard.server.gen.transport.TransportProtos.AlarmStateProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; @@ -45,12 +46,16 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingZoneState; +import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.TreeMap; @@ -92,7 +97,10 @@ public class CalculatedFieldUtils { .setType(state.getType().name()); state.getArguments().forEach((argName, argEntry) -> { - if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + if (argEntry instanceof AggArgumentEntry aggArgumentEntry) { + aggArgumentEntry.getAggInputs() + .forEach((entityId, entry) -> builder.addAggArguments(toAggSingleArgumentProto(argName, entityId, entry))); + } else if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { builder.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); @@ -130,6 +138,17 @@ public class CalculatedFieldUtils { return ruleState; } + public static AggSingleArgumentEntryProto toAggSingleArgumentProto(String argName, EntityId entityId, ArgumentEntry argumentEntry) { + AggSingleArgumentEntryProto.Builder builder = AggSingleArgumentEntryProto.newBuilder() + .setEntityId(ProtoUtils.toProto(entityId)); + + if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + builder.setValue(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); + } + + return builder.build(); + } + public static SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() .setArgName(argName); @@ -187,6 +206,7 @@ public class CalculatedFieldUtils { case SCRIPT -> new ScriptCalculatedFieldState(id.entityId()); case GEOFENCING -> new GeofencingCalculatedFieldState(id.entityId()); case ALARM -> new AlarmCalculatedFieldState(id.entityId()); + case LATEST_VALUES_AGGREGATION -> new LatestValuesAggregationCalculatedFieldState(id.entityId()); }; proto.getSingleValueArgumentsList().forEach(argProto -> @@ -212,11 +232,37 @@ public class CalculatedFieldUtils { alarmState.setClearRuleState(fromAlarmRuleStateProto(alarmStateProto.getClearRuleState(), alarmState)); } } + case LATEST_VALUES_AGGREGATION -> { + LatestValuesAggregationCalculatedFieldState aggState = (LatestValuesAggregationCalculatedFieldState) state; + Map> arguments = new HashMap<>(); + proto.getAggArgumentsList().forEach(argProto -> { + AggSingleArgumentEntry entry = fromAggSingleValueArgumentProto(argProto); + arguments.computeIfAbsent(argProto.getValue().getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); + }); + arguments.forEach((argName, entityInputs) -> { + aggState.getArguments().put(argName, new AggArgumentEntry(entityInputs, false)); + }); + } } return state; } + public static AggSingleArgumentEntry fromAggSingleValueArgumentProto(AggSingleArgumentEntryProto proto) { + if (!proto.hasValue()) { + return new AggSingleArgumentEntry(); + } + EntityId entityId = ProtoUtils.fromProto(proto.getEntityId()); + SingleValueArgumentProto singleValueArgument = proto.getValue(); + TsValueProto tsValueProto = singleValueArgument.getValue(); + return new AggSingleArgumentEntry( + entityId, + tsValueProto.getTs(), + (BasicKvEntry) KvProtoUtil.fromTsValueProto(singleValueArgument.getArgName(), tsValueProto), + singleValueArgument.getVersion() + ); + } + public static SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { if (!proto.hasValue()) { return new SingleValueArgumentEntry(); diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java new file mode 100644 index 0000000000..1a5fc2e3d3 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -0,0 +1,314 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cf; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; +import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.data.DeviceData; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationPathLevel; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.thingsboard.server.cf.CalculatedFieldIntegrationTest.POLL_INTERVAL; + +@DaoSqlTest +public class LatestValuesAggregationCalculatedFieldTest extends AbstractControllerTest { + + private Tenant savedTenant; + + private DeviceProfile deviceProfile; + private Device device1; + private String accessToken1 = "1234567890111"; + private Device device2; + private String accessToken2 = "1234567890222"; + + private AssetProfile assetProfile; + private Asset asset; + + private CalculatedField calculatedField; + + private long deduplicationInterval = 10000; + + @Before + public void beforeTest() throws Exception { + loginSysAdmin(); + + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + savedTenant = saveTenant(tenant); + assertThat(savedTenant).isNotNull(); + + User tenantAdmin = new User(); + tenantAdmin.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin.setTenantId(savedTenant.getId()); + tenantAdmin.setEmail("tenant@thingsboard.org"); + tenantAdmin.setFirstName("John"); + tenantAdmin.setLastName("Doe"); + + createUserAndLogin(tenantAdmin, "testPassword"); + + deviceProfile = doPost("/api/deviceProfile", createDeviceProfile("Device Profile"), DeviceProfile.class); + device1 = createDevice("Device 1", deviceProfile.getId(), accessToken1); + device2 = createDevice("Device 2", deviceProfile.getId(), accessToken2); + + postTelemetry(device1.getId(), "{\"occupied\":true}"); + postTelemetry(device2.getId(), "{\"occupied\":false}"); + + assetProfile = doPost("/api/assetProfile", createAssetProfile("Asset Profile"), AssetProfile.class); + asset = createAsset("Asset", assetProfile.getId()); + + createEntityRelation(asset.getId(), device1.getId(), "Contains"); + createEntityRelation(asset.getId(), device2.getId(), "Contains"); + + calculatedField = createOccupancyCF(asset.getId(), List.of(deviceProfile.getId())); + + checkInitialCalculation(); + } + + @After + public void afterTest() throws Exception { + loginSysAdmin(); + + deleteTenant(savedTenant.getId()); + } + + @Test + public void testUpdateTelemetry_checkMetricsCalculation() throws Exception { + postTelemetry(device1.getId(), "{\"occupied\":false}"); + + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + }); + } + + @Test + public void testUpdateTelemetry_checkMetricsCalculationNotExecutedUntilDeduplicationInterval() throws Exception { + postTelemetry(device1.getId(), "{\"occupied\":false}"); + + await().alias("update telemetry -> no changes").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(this::checkInitialCalculationValues); + + postTelemetry(device2.getId(), "{\"occupied\":false}"); + + await().alias("create CF and perform initial calculation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + }); + } + + @Test + public void testChangeProfile_checkMetricsCalculation() throws Exception { + DeviceProfile deviceProfile2 = doPost("/api/deviceProfile", createDeviceProfile("Device Profile 2"), DeviceProfile.class); + device1.setDeviceProfileId(deviceProfile2.getId()); + device1 = doPost("/api/device?accessToken=" + accessToken1, device1, Device.class); + + postTelemetry(device1.getId(), "{\"occupied\":false}"); + + await().alias("change profile and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); + }); + } + + @Test + public void testAddEntityToProfile_checkMetricsCalculation() throws Exception { + Device device3 = createDevice("Device 3", deviceProfile.getId(), "1234567890333"); + + postTelemetry(device3.getId(), "{\"occupied\":true}"); + + await().alias("add entity to profile and no calculation (there is no relation between device and asset)").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(this::checkInitialCalculationValues); + + createEntityRelation(asset.getId(), device3.getId(), "Contains"); + + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); + }); + } + + @Test + public void testDeleteRelation_checkMetricsCalculation() throws Exception { + deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); + + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); + }); + } + + private void checkInitialCalculation() { + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(this::checkInitialCalculationValues); + } + + private void checkInitialCalculationValues() throws Exception { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + } + + private CalculatedField createOccupancyCF(EntityId entityId, List profiles) { + Map aggMetrics = new HashMap<>(); + + AggMetric freeSpaces = new AggMetric(); + freeSpaces.setFunction(AggFunction.COUNT); + freeSpaces.setFilter("return oc == false"); + freeSpaces.setInput(new AggKeyInput("oc")); + aggMetrics.put("freeSpaces", freeSpaces); + + AggMetric occupiedSpaces = new AggMetric(); + occupiedSpaces.setFunction(AggFunction.COUNT); + occupiedSpaces.setFilter("return oc == true"); + occupiedSpaces.setInput(new AggKeyInput("oc")); + aggMetrics.put("occupiedSpaces", occupiedSpaces); + + AggMetric totalSpaces = new AggMetric(); + totalSpaces.setFunction(AggFunction.COUNT); + totalSpaces.setInput(new AggFunctionInput("return 1;")); + aggMetrics.put("totalSpaces", totalSpaces); + + Output output = new Output(); + output.setType(OutputType.TIME_SERIES); + + return createAggCf("Occupied spaces", entityId, + buildSource(EntitySearchDirection.FROM, "Contains", profiles), + Map.of("oc", new ReferencedEntityKey("occupied", ArgumentType.TS_LATEST, null)), + aggMetrics, + output); + } + + private AggSource buildSource(EntitySearchDirection direction, String relationType, List profiles) { + AggSource source = new AggSource(); + source.setRelation(new RelationPathLevel(direction, relationType)); + source.setEntityProfiles(profiles); + return source; + } + + private CalculatedField createAggCf(String name, + EntityId entityId, + AggSource aggSource, + Map inputs, + Map metrics, + Output output) { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setName(name); + calculatedField.setEntityId(entityId); + calculatedField.setType(CalculatedFieldType.LATEST_VALUES_AGGREGATION); + + LatestValuesAggregationCalculatedFieldConfiguration configuration = new LatestValuesAggregationCalculatedFieldConfiguration(); + configuration.setSource(aggSource); + configuration.setInputs(inputs); + configuration.setDeduplicationIntervalMillis(deduplicationInterval); + configuration.setMetrics(metrics); + configuration.setOutput(output); + + calculatedField.setConfiguration(configuration); + calculatedField.setDebugSettings(DebugSettings.all()); + return saveCalculatedField(calculatedField); + } + + private Device createDevice(String name, DeviceProfileId deviceProfileId, String accessToken) { + Device device = new Device(); + device.setName(name); + device.setDeviceProfileId(deviceProfileId); + DeviceData deviceData = new DeviceData(); + deviceData.setTransportConfiguration(new DefaultDeviceTransportConfiguration()); + deviceData.setConfiguration(new DefaultDeviceConfiguration()); + device.setDeviceData(deviceData); + return doPost("/api/device?accessToken=" + accessToken, device, Device.class); + } + + private Asset createAsset(String name, AssetProfileId assetProfileId) { + Asset asset = new Asset(); + asset.setName(name); + asset.setAssetProfileId(assetProfileId); + return doPost("/api/asset", asset, Asset.class); + } + + private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 2050e1075c..6c6b9c44ec 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -1063,6 +1063,16 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { doPost("/api/relation", relation); } + protected void deleteEntityRelation(EntityRelation entityRelation) throws Exception { + String url = String.format("/api/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", + entityRelation.getFrom().getId(), + entityRelation.getFrom().getEntityType(), + entityRelation.getType(), + entityRelation.getTo().getId(), + entityRelation.getTo().getEntityType()); + doDelete(url); + } + protected List findRelationsByTo(EntityId entityId) throws Exception { String url = String.format("/api/relations?toId=%s&toType=%s", entityId.getId(), entityId.getEntityType().name()); MvcResult mvcResult = doGet(url).andReturn(); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 701c22587b..6c8acbfb83 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; @@ -30,6 +31,7 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; @@ -87,6 +89,8 @@ public interface TbClusterService extends TbQueueClusterService { void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); + void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, EntityId profileId, ComponentLifecycleEvent state); + void broadcast(ComponentLifecycleMsg componentLifecycleMsg); void onDeviceProfileChange(DeviceProfile deviceProfile, DeviceProfile oldDeviceProfile, TbQueueCallback callback); @@ -137,4 +141,8 @@ public interface TbClusterService extends TbQueueClusterService { void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback); + void onRelationUpdated(TenantId tenantId, EntityRelation entityRelation, TbQueueCallback callback); + + void onRelationDeleted(TenantId tenantId, EntityRelation entityRelation, TbQueueCallback callback); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index a0bc9a72e6..5b3290c110 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -86,6 +86,14 @@ public interface RelationService { ListenableFuture> findByRelationPathQueryAsync(TenantId tenantId, EntityRelationPathQuery relationPathQuery); + List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId profileId); + + EntityRelation findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId); + + void evictRelationsByProfile(TenantId tenantId, EntityId profileId); + + void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId); + // TODO: This method may be useful for some validations in the future // ListenableFuture checkRecursiveRelation(EntityId from, EntityId to); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index 7052ab70e9..397d31650a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -24,7 +24,8 @@ public enum CalculatedFieldType { SIMPLE, SCRIPT, GEOFENCING, - ALARM; + ALARM, + LATEST_VALUES_AGGREGATION; public static final Set all = Collections.unmodifiableSet(EnumSet.allOf(CalculatedFieldType.class)); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index d3622a2dcf..4a4e3380f5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -40,7 +41,8 @@ import java.util.stream.Collectors; @Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), @Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT"), @Type(value = GeofencingCalculatedFieldConfiguration.class, name = "GEOFENCING"), - @Type(value = AlarmCalculatedFieldConfiguration.class, name = "ALARM") + @Type(value = AlarmCalculatedFieldConfiguration.class, name = "ALARM"), + @Type(value = LatestValuesAggregationCalculatedFieldConfiguration.class, name = "LATEST_VALUES_AGGREGATION") }) @JsonIgnoreProperties(ignoreUnknown = true) public interface CalculatedFieldConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunction.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunction.java new file mode 100644 index 0000000000..cd0e3f66d4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunction.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +public enum AggFunction { + MIN, MAX, SUM, AVG, COUNT, COUNT_UNIQUE +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunctionInput.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunctionInput.java new file mode 100644 index 0000000000..f6df80952e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunctionInput.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class AggFunctionInput implements AggInput { + + private String function; + + @Override + public String getType() { + return "function"; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java new file mode 100644 index 0000000000..fac988d5d9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = AggKeyInput.class, name = "key"), + @JsonSubTypes.Type(value = AggFunctionInput.class, name = "function") +}) +@JsonIgnoreProperties(ignoreUnknown = true) +public interface AggInput { + + String getType(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggKeyInput.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggKeyInput.java new file mode 100644 index 0000000000..1a00a18a9d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggKeyInput.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class AggKeyInput implements AggInput { + + private String key; + + @Override + public String getType() { + return "key"; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java new file mode 100644 index 0000000000..ebd612b1e0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Data; + +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class AggMetric { + + private AggFunction function; + private String filter; + private AggInput input; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java new file mode 100644 index 0000000000..84f6f92eb3 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.RelationPathLevel; + +import java.util.List; + +@Data +public class AggSource { + + private RelationPathLevel relation; + private List entityProfiles; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java new file mode 100644 index 0000000000..393697877e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java @@ -0,0 +1,104 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; +import java.util.function.Predicate; + +@Data +@Builder +public class CfAggTrigger { + + private List entityProfiles; + private List inputs; + + public boolean matches(EntityId profileId, Predicate cfAggTrigger) { + if (matchesProfile(profileId)) { + return cfAggTrigger.test(this); + } + return false; + } + + public boolean matchesProfile(EntityId profileId) { + return entityProfiles.contains(profileId); + } + + public boolean matchesTimeSeries(List telemetry) { + if (telemetry == null || telemetry.isEmpty()) { + return false; + } + for (TsKvEntry tsKvEntry : telemetry) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKvEntry.getKey(), ArgumentType.TS_LATEST, null); + if (inputs.contains(latestKey)) { + return true; + } + } + return false; + } + + public boolean matchesAttributes(List attributes, AttributeScope scope) { + if (attributes == null || attributes.isEmpty()) { + return false; + } + for (AttributeKvEntry attributeKvEntry : attributes) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(attributeKvEntry.getKey(), ArgumentType.ATTRIBUTE, scope); + if (inputs.contains(latestKey)) { + return true; + } + } + return false; + } + + public boolean matchesTimeSeriesKeys(List telemetry) { + if (telemetry == null || telemetry.isEmpty()) { + return false; + } + + for (String key : telemetry) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); + if (inputs.contains(latestKey)) { + return true; + } + } + + return false; + } + + public boolean matchesAttributesKeys(List attributes, AttributeScope scope) { + if (attributes == null || attributes.isEmpty()) { + return false; + } + + for (String key : attributes) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); + if (inputs.contains(latestKey)) { + return true; + } + } + + return false; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..e3db50fda0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; + +import java.util.List; +import java.util.Map; + +@Data +public class LatestValuesAggregationCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + private AggSource source; + private Map inputs; + private long deduplicationIntervalMillis; + private Map metrics; + private Output output; + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.LATEST_VALUES_AGGREGATION; + } + + @Override + public void validate() { + } + + public CfAggTrigger buildTrigger() { + return CfAggTrigger.builder() + .inputs(List.copyOf(inputs.values())) + .entityProfiles(source.getEntityProfiles()) + .build(); + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index c13b0200c7..f82de35819 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -152,6 +152,8 @@ public enum MsgType { CF_ENTITY_INIT_CF_MSG, CF_ENTITY_DELETE_MSG, + CF_RELATED_ENTITY_MSG, + CF_ARGUMENT_RESET_MSG, // Sent to reset argument; CF_REEVALUATE_MSG; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index 23b9fe08e3..38df529c96 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -47,14 +47,15 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { private final EntityId oldProfileId; private final EntityId profileId; private final boolean ownerChanged; + private final boolean relationChanged; private final JsonNode info; public ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event) { - this(tenantId, entityId, event, null, null, null, null, false, null); + this(tenantId, entityId, event, null, null, null, null, false, false, null); } @Builder - private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId, boolean ownerChanged, JsonNode info) { + private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId, boolean ownerChanged, boolean relationChanged, JsonNode info) { this.tenantId = tenantId; this.entityId = entityId; this.event = event; @@ -63,6 +64,7 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { this.oldProfileId = oldProfileId; this.profileId = profileId; this.ownerChanged = ownerChanged; + this.relationChanged = relationChanged; this.info = info; } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 26a64c7f8a..83fb158efd 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -130,6 +130,7 @@ public class ProtoUtils { builder.setOldProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); } builder.setOwnerChanged(msg.isOwnerChanged()); + builder.setRelationChanged(msg.isRelationChanged()); if (msg.getName() != null) { builder.setName(msg.getName()); } @@ -167,6 +168,7 @@ public class ProtoUtils { builder.oldProfileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB()))); } builder.ownerChanged(proto.getOwnerChanged()); + builder.relationChanged(proto.getRelationChanged()); if (proto.hasInfo()) { builder.info(JacksonUtil.toJsonNode(proto.getInfo())); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 4d99608a6d..764f4c3ec9 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -915,6 +915,11 @@ message GeofencingArgumentProto { repeated GeofencingZoneProto zones = 2; } +message AggSingleArgumentEntryProto { + EntityIdProto entityId = 1; + SingleValueArgumentProto value = 2; +} + message CalculatedFieldStateProto { CalculatedFieldEntityCtxIdProto id = 1; string type = 2; @@ -922,6 +927,7 @@ message CalculatedFieldStateProto { repeated TsRollingArgumentProto rollingValueArguments = 4; repeated GeofencingArgumentProto geofencingArguments = 5; AlarmStateProto alarmState = 6; + repeated AggSingleArgumentEntryProto aggArguments = 7; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. @@ -1292,6 +1298,7 @@ message ComponentLifecycleMsgProto { int64 profileIdLSB = 12; optional string info = 13; bool ownerChanged = 100; + bool relationChanged = 15; } message EdgeEventMsgProto { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java index 73a2183564..bf2123a26b 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java @@ -28,6 +28,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = TbelCfSingleValueArg.class, name = "SINGLE_VALUE"), @JsonSubTypes.Type(value = TbelCfTsRollingArg.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = TbelCfTsGeofencingArg.class, name = "GEOFENCING_CF_ARGUMENT_VALUE"), + @JsonSubTypes.Type(value = TbelCfLatestValuesAggregation.class, name = "LATEST_VALUES_AGGREGATION") }) public interface TbelCfArg extends TbelCfObject { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java new file mode 100644 index 0000000000..1b5fa394d2 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class TbelCfLatestValuesAggregation implements TbelCfArg { + + private final Object value; + + @JsonCreator + public TbelCfLatestValuesAggregation( + @JsonProperty("value") Object value + ) { + this.value = value; + } + + + @Override + public String getType() { + return "LATEST_VALUES_AGGREGATION"; + } + + @Override + public long memorySize() { + return 32; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 18d1806fe4..df79ca145a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -514,6 +514,40 @@ public class BaseRelationService implements RelationService { return executor.submit(() -> relationDao.findByRelationPathQuery(tenantId, relationPathQuery)); } + @Override + public List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId profileId) { + RelationCacheKey cacheKey = RelationCacheKey.builder().from(from).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.FROM).entityProfile(profileId).build(); + return cache.getAndPutInTransaction(cacheKey, + () -> relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, profileId), + RelationCacheValue::getRelations, + relations -> RelationCacheValue.builder().relations(relations).build(), false); + } + + @Override + public EntityRelation findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId) { + RelationCacheKey cacheKey = RelationCacheKey.builder().to(to).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.TO).entityProfile(profileId).build(); + return cache.getAndPutInTransaction(cacheKey, + () -> relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, profileId), + RelationCacheValue::getRelation, + relation -> RelationCacheValue.builder().relation(relation).build(), false); + } + + @Override + public void evictRelationsByProfile(TenantId tenantId, EntityId profileId) { + RelationCacheKey key = RelationCacheKey.builder().entityProfile(profileId).build(); + cache.evict(List.of(key)); + log.debug("Processed evict relations by key: {}", key); + } + + @Override + public void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId) { + List keys = new ArrayList<>(2); + keys.add(RelationCacheKey.builder().from(entityId).entityProfile(profileId).build()); + keys.add(RelationCacheKey.builder().to(entityId).entityProfile(profileId).build()); + cache.evict(keys); + log.debug("Processed evict relations by keys: {}", keys); + } + private void validate(EntityRelationPathQuery relationPathQuery) { validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); List levels = relationPathQuery.levels(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java index d6f0525c9d..344af0a6f3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java @@ -40,9 +40,14 @@ public class RelationCacheKey implements Serializable { private final String type; private final RelationTypeGroup typeGroup; private final EntitySearchDirection direction; + private final EntityId entityProfile; public RelationCacheKey(EntityId from, EntityId to, String type, RelationTypeGroup typeGroup) { - this(from, to, type, typeGroup, null); + this(from, to, type, typeGroup, null, null); + } + + public RelationCacheKey(EntityId from, EntityId to, String type, RelationTypeGroup typeGroup, EntitySearchDirection direction) { + this(from, to, type, typeGroup, direction, null); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index ad53164ad7..f529396965 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -36,8 +36,12 @@ public interface RelationDao { List findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup); + List findByFromAndTypeAndProfile(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup, EntityId profileId); + List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup); + EntityRelation findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId); + List findAllByTo(TenantId tenantId, EntityId to); List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index b2871313ed..c859b71580 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -103,6 +103,11 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override + public List findByFromAndTypeAndProfile(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { + return DaoUtil.convertDataList(relationRepository.findByFromAndProfile(from.getId(), from.getEntityType().name(), typeGroup.name(), relationType, profileId.getId())); + } + @Override public List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup) { return DaoUtil.convertDataList( @@ -112,6 +117,11 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override + public EntityRelation findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { + return DaoUtil.getData(relationRepository.findByToAndProfile(to.getId(), to.getEntityType().name(), typeGroup.name(), relationType, profileId.getId())); + } + @Override public List findAllByTo(TenantId tenantId, EntityId to) { return DaoUtil.convertDataList( diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index b4e8a21372..9294236526 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.sql.relation; -import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; @@ -28,6 +27,7 @@ import org.thingsboard.server.dao.model.sql.RelationCompositeKey; import org.thingsboard.server.dao.model.sql.RelationEntity; import java.util.List; +import java.util.Optional; import java.util.UUID; public interface RelationRepository @@ -96,4 +96,41 @@ public interface RelationRepository @Param("toId") UUID toId, @Param("toType") String toType, @Param("batchSize") int batchSize); + + @Query(value = """ + SELECT r.from_id, r.from_type, r.relation_type_group, r.relation_type, r.to_id, r.to_type, r.additional_info, r.version + FROM relation r + LEFT JOIN device d ON r.to_id = d.id AND r.to_type = 'DEVICE' + LEFT JOIN asset a ON r.to_id = a.id AND r.to_type = 'ASSET' + WHERE r.from_id = :fromId + AND r.from_type = :fromType + AND r.relation_type = :relationType + AND r.relation_type_group = :relationTypeGroup + AND ((d.device_profile_id = :profileId) OR (a.asset_profile_id = :profileId)) + AND (d.id IS NOT NULL OR a.id IS NOT NULL) + """, nativeQuery = true) + List findByFromAndProfile(@Param("fromId") UUID fromId, + @Param("fromType") String fromType, + @Param("relationTypeGroup") String relationTypeGroup, + @Param("relationType") String relationType, + @Param("profileId") UUID profileId); + + @Query(value = """ + SELECT r.from_id, r.from_type, r.relation_type_group, r.relation_type, r.to_id, r.to_type, r.additional_info, r.version + FROM relation r + LEFT JOIN device d ON r.from_id = d.id AND r.from_type = 'DEVICE' + LEFT JOIN asset a ON r.from_id = a.id AND r.from_type = 'ASSET' + WHERE r.to_id = :toId + AND r.to_type = :toType + AND r.relation_type = :relationType + AND r.relation_type_group = :relationTypeGroup + AND ((d.device_profile_id = :profileId) OR (a.asset_profile_id = :profileId)) + AND (d.id IS NOT NULL OR a.id IS NOT NULL) + LIMIT 1 + """, nativeQuery = true) + Optional findByToAndProfile(@Param("toId") UUID toId, + @Param("toType") String toType, + @Param("relationTypeGroup") String relationTypeGroup, + @Param("relationType") String relationType, + @Param("profileId") UUID profileId); } From a585a3b9e69ec1c0cad03c96424db65301caa171 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 14 Oct 2025 13:03:45 +0300 Subject: [PATCH 02/33] minor refactoring --- ...CalculatedFieldEntityMessageProcessor.java | 4 +- ...alculatedFieldManagerMessageProcessor.java | 86 ++++------ ...tractCalculatedFieldProcessingService.java | 158 ++++++++++-------- .../cf/CalculatedFieldProcessingService.java | 2 +- ...faultCalculatedFieldProcessingService.java | 4 +- .../DefaultCalculatedFieldQueueService.java | 28 +++- .../state/aggregation/AggArgumentEntry.java | 3 +- .../service/cf/ctx/state/aggregation/agg.json | 4 +- .../processing/AbstractConsumerService.java | 2 - ...tValuesAggregationCalculatedFieldTest.java | 73 +++++++- .../server/dao/relation/RelationService.java | 11 +- .../aggregation/CfAggTrigger.java | 2 +- ...gregationCalculatedFieldConfiguration.java | 2 +- .../ProfileEntityRelationPathQuery.java | 21 +++ .../tbel/TbelCfLatestValuesAggregation.java | 2 +- .../dao/relation/BaseRelationService.java | 98 +++++++++-- .../server/dao/relation/RelationDao.java | 5 +- .../dao/sql/relation/JpaRelationDao.java | 102 ++++++++++- .../dao/sql/relation/RelationRepository.java | 3 +- 19 files changed, 436 insertions(+), 174 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 98b5eba4c8..9dfde4d821 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -69,9 +69,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createStateByType; @@ -264,7 +262,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM @SneakyThrows private Map fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { - ListenableFuture> argumentsFuture = cfService.fetchAggArguments(ctx, entityId); + ListenableFuture> argumentsFuture = cfService.fetchAggEntityArguments(ctx, entityId); // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 8a717c3cdb..3c1e21bcc8 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -24,8 +24,8 @@ import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.calculatedField.EntityInitCalculatedFieldMsg.StateAction; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; -import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ProfileEntityIdInfo; @@ -255,7 +255,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); // process aggregation cfs(in any) - List cfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(entityId, profileId); + List cfsRelatedToEntity = getCfsWithRelationToEntity(entityId, profileId); if (!cfsRelatedToEntity.isEmpty()) { MultipleTbCallback multiCallback = new MultipleTbCallback(cfsRelatedToEntity.size(), callbackFor2); cfsRelatedToEntity.forEach(ctx -> { @@ -289,8 +289,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); // process aggregation cfs(in any) - List oldCfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getOldProfileId()); - List newCfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getProfileId()); + List oldCfsRelatedToEntity = getCfsWithRelationToEntity(msg.getEntityId(), msg.getOldProfileId()); + List newCfsRelatedToEntity = getCfsWithRelationToEntity(msg.getEntityId(), msg.getProfileId()); var fieldsWithRelatedEntityCount = oldCfsRelatedToEntity.size() + newCfsRelatedToEntity.size(); if (fieldsWithRelatedEntityCount > 0) { MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsWithRelatedEntityCount, callbackFor2); @@ -335,7 +335,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); - getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getProfileId()).forEach(ctx -> { + getCfsWithRelationToEntity(msg.getEntityId(), msg.getProfileId()).forEach(ctx -> { applyToTargetCfEntityActors(ctx, callback, (id, cb) -> deleteRelatedEntity(id, msg.getEntityId(), cb)); }); if (isMyPartition(msg.getEntityId(), callback)) { @@ -559,6 +559,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onCfDeleted(ComponentLifecycleMsg msg, TbCallback callback) { var cfId = new CalculatedFieldId(msg.getEntityId().getId()); var cfCtx = calculatedFields.remove(cfId); // fixme wtf? why isn't ctx closed properly? + cfTriggers.remove(cfId); if (cfCtx == null) { log.debug("[{}] CF was already deleted [{}]", tenantId, cfId); callback.onSuccess(); @@ -613,76 +614,55 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private List filterAggregationCfs(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); - - List aggregationCalculatedFields = cfTriggers.entrySet().stream() + return cfTriggers.entrySet().stream() .filter(entry -> aggMatches(entry.getValue(), msg.getProto())) .map(Entry::getKey) .map(calculatedFields::get) .filter(Objects::nonNull) + .flatMap(cf -> findRelationsForCf(entityId, cf).stream()) .toList(); - - List filteredByRelationCfs = new ArrayList<>(); - for (CalculatedFieldCtx cf : aggregationCalculatedFields) { - EntityId cfEntityId = cf.getEntityId(); - if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - RelationPathLevel relation = aggConfig.getSource().getRelation(); - EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) - ? cfEntityId - : getProfileId(tenantId, cfEntityId); - EntityId targetEntity = switch (relation.direction()) { - case FROM -> - relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).getFrom(); - case TO -> - relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0).getTo(); - }; - if (targetEntity != null) { - filteredByRelationCfs.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), targetEntity)); - } - } - } - return filteredByRelationCfs; } - private List getCalculatedFieldsRelatedToEntity(EntityId entityId, EntityId profileId) { - List aggCFsUsedProfile = cfTriggers.entrySet().stream() + private List getCfsWithRelationToEntity(EntityId entityId, EntityId profileId) { + return cfTriggers.entrySet().stream() .filter(entry -> entry.getValue().matchesProfile(profileId)) .map(Entry::getKey) .map(calculatedFields::get) .filter(Objects::nonNull) + .filter(cf -> !findRelationsForCf(entityId, cf).isEmpty()) .toList(); - - List filteredByRelationCfs = new ArrayList<>(); - for (CalculatedFieldCtx cf : aggCFsUsedProfile) { - CalculatedFieldEntityCtxId calculatedFieldEntityCtxId = filterCfByRelationWithEntity(entityId, cf); - if (calculatedFieldEntityCtxId != null) { - filteredByRelationCfs.add(cf); - } - } - return filteredByRelationCfs; } - private CalculatedFieldEntityCtxId filterCfByRelationWithEntity(EntityId entityId, CalculatedFieldCtx cf) { - EntityId cfEntityId = cf.getEntityId(); - if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - RelationPathLevel relation = aggConfig.getSource().getRelation(); - EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) + private List findRelationsForCf(EntityId entityId, CalculatedFieldCtx cf) { + List result = new ArrayList<>(); + if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration configuration) { + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + EntityId cfEntityId = cf.getEntityId(); + EntityId targetProfileId = isProfileEntity(cfEntityId.getEntityType()) ? cfEntityId : getProfileId(tenantId, cfEntityId); - EntityId targetEntity = switch (relation.direction()) { + switch (relation.direction()) { case FROM -> { - EntityRelation entityRelation = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); - yield entityRelation == null ? null : entityRelation.getFrom(); + List relationsByTo = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), targetProfileId); + if (relationsByTo != null && !relationsByTo.isEmpty()) { + EntityRelation entityRelation = relationsByTo.get(0); // only one supported + result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getFrom())); + } } case TO -> { - EntityRelation entityRelation = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0); - yield entityRelation == null ? null : entityRelation.getTo(); + List relationsByFrom = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), targetProfileId); + if (relationsByFrom != null && !relationsByFrom.isEmpty()) { + for (EntityRelation entityRelation : relationsByFrom) { + if (entityRelation.getTo().equals(cf.getEntityId())) { + result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getTo())); + } + } + } } - }; - if (targetEntity != null) { - return new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), targetEntity); } } - return null; + return result; } private boolean aggMatches(CfAggTrigger cfAggTrigger, CalculatedFieldTelemetryMsgProto proto) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 304b747f26..db15fc1dc8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -26,7 +26,6 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; @@ -40,6 +39,7 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.attributes.AttributesService; @@ -51,10 +51,11 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; -import java.util.Collection; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -105,7 +106,7 @@ public abstract class AbstractCalculatedFieldProcessingService { } yield futures; } - case LATEST_VALUES_AGGREGATION -> fetchAggregationArgumentFutures(ctx, entityId); + case LATEST_VALUES_AGGREGATION -> fetchAggArguments(ctx, entityId, ts); }; return Futures.whenAllComplete(argFutures.values()) .call(() -> resolveArgumentFutures(argFutures), @@ -122,17 +123,32 @@ public abstract class AbstractCalculatedFieldProcessingService { return resolveOwnerArgument(tenantId, entityId); } - private List resolveRelatedEntities(TenantId tenantId, EntityId entityId, AggSource aggSource) { + private ListenableFuture> resolveRelatedEntities(TenantId tenantId, EntityId entityId, AggSource aggSource) { RelationPathLevel relation = aggSource.getRelation(); - return switch (relation.direction()) { - case FROM -> aggSource.getEntityProfiles().stream() - .map(profile -> relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), profile)) - .flatMap(Collection::stream) - .map(EntityRelation::getTo) + + List>> relationListsFut = new ArrayList<>(); + if (aggSource.getEntityProfiles().isEmpty()) { + relationListsFut.add(relationService.findByProfileEntityRelationPathQueryAsync(tenantId, new ProfileEntityRelationPathQuery(entityId, relation, null))); + } else { + aggSource.getEntityProfiles().forEach(profile -> relationListsFut.add(relationService.findByProfileEntityRelationPathQueryAsync(tenantId, new ProfileEntityRelationPathQuery(entityId, relation, profile)))); + } + + return Futures.transform(Futures.allAsList(relationListsFut), relationLists -> { + if (relationLists == null) { + return new ArrayList<>(); + } + List allRelations = relationLists.stream() + .filter(Objects::nonNull) + .flatMap(List::stream) .toList(); - case TO -> - aggSource.getEntityProfiles().stream().map(profile -> relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), profile).getFrom()).toList(); - }; + + return switch (relation.direction()) { + case FROM -> allRelations.stream() + .map(EntityRelation::getTo) + .toList(); + case TO -> allRelations.isEmpty() ? List.of() : List.of(allRelations.get(0).getFrom()); + }; + }, calculatedFieldCallbackExecutor); } protected Map resolveArgumentFutures(Map> argFutures) { @@ -174,6 +190,34 @@ public abstract class AbstractCalculatedFieldProcessingService { return argFutures; } + protected Map> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { + LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + + ListenableFuture> relatedEntities = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getSource()); + + Map> futures = new HashMap<>(); + aggConfig.getInputs().forEach((key, refKey) -> { + Argument argument = new Argument(); + argument.setRefEntityKey(refKey); + futures.put(key, Futures.transformAsync(relatedEntities, entityIds -> fetchAggArgumentEntry(ctx.getTenantId(), entityIds, argument, System.currentTimeMillis()), MoreExecutors.directExecutor())); + }); + return futures; + } + + protected ListenableFuture> fetchEntityAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { + LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + Map> futures = new HashMap<>(); + aggConfig.getInputs().forEach((key, refKey) -> { + Argument argument = new Argument(); + argument.setRefEntityKey(refKey); + ListenableFuture argEntryFut = fetchSingleAggArgumentEntry(ctx.getTenantId(), entityId, argument, ts); + futures.put(key, argEntryFut); + }); + return Futures.whenAllComplete(futures.values()) + .call(() -> resolveArgumentFutures(futures), + MoreExecutors.directExecutor()); + } + private ListenableFuture> resolveGeofencingEntityIds(TenantId tenantId, EntityId entityId, Map.Entry entry) { Argument value = entry.getValue(); if (value.getRefEntityId() != null) { @@ -197,50 +241,6 @@ public abstract class AbstractCalculatedFieldProcessingService { return ownerService.getOwner(tenantId, entityId); } - private Map> fetchAggregationArgumentFutures(CalculatedFieldCtx ctx, EntityId entityId) { - LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - - List entityIds = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getSource()); - - Map> futures = new HashMap<>(); - aggConfig.getInputs().forEach((key, refKey) -> { - Argument argument = new Argument(); - argument.setRefEntityKey(refKey); - futures.put(key, fetchAggArgumentEntry(ctx.getTenantId(), entityIds, argument, System.currentTimeMillis())); - }); - return futures; - } - - public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) { - List>> futures = aggEntities.stream() - .map(entityId -> fetchSingleAggArgumentEntry(tenantId, entityId, argument, startTs)) - .toList(); - - ListenableFuture>> allFutures = Futures.allAsList(futures); - - return Futures.transform(allFutures, - entries -> ArgumentEntry.createAggArgument( - entries.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) - ), - MoreExecutors.directExecutor()); - } - - protected ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { - CalculatedFieldConfiguration configuration = ctx.getCalculatedField().getConfiguration(); - LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) configuration; - Map> futures = new HashMap<>(); - aggConfig.getInputs().forEach((key, refKey) -> { - Argument argument = new Argument(); - argument.setRefEntityKey(refKey); - - ListenableFuture argumentEntryListenableFuture = fetchAggArgumentEntry(ctx.getTenantId(), List.of(entityId), argument, System.currentTimeMillis()); - futures.put(key, argumentEntryListenableFuture); - }); - return Futures.whenAllComplete(futures.values()) - .call(() -> resolveArgumentFutures(futures), - MoreExecutors.directExecutor()); - } - private ListenableFuture fetchGeofencingKvEntry(TenantId tenantId, List geofencingEntities, Argument argument) { if (argument.getRefEntityKey().getType() != ArgumentType.ATTRIBUTE) { throw new IllegalStateException("Unsupported argument key type: " + argument.getRefEntityKey().getType()); @@ -265,6 +265,22 @@ public abstract class AbstractCalculatedFieldProcessingService { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))), MoreExecutors.directExecutor()); } + public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) {List>> futures = aggEntities.stream() + .map(entityId -> { + ListenableFuture singleAggEntryFut = fetchSingleAggArgumentEntry(tenantId, entityId, argument, startTs); + return Futures.transform(singleAggEntryFut, singleAggEntry -> Map.entry(entityId, singleAggEntry), MoreExecutors.directExecutor()); + }) + .toList(); + + ListenableFuture>> allFutures = Futures.allAsList(futures); + + return Futures.transform(allFutures, + entries -> ArgumentEntry.createAggArgument( + entries.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) + ), + MoreExecutors.directExecutor()); + } + protected ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { return switch (argument.getRefEntityKey().getType()) { case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument, startTs); @@ -309,7 +325,15 @@ public abstract class AbstractCalculatedFieldProcessingService { }, calculatedFieldCallbackExecutor)); } - protected ListenableFuture> fetchSingleAggArgumentEntry(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { + private ReadTsKvQuery buildTsRollingQuery(TenantId tenantId, Argument argument, long startTs, long endTs) { + long maxDataPoints = apiLimitService.getLimit( + tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); + int argumentLimit = argument.getLimit(); + int limit = argumentLimit == 0 || argumentLimit > maxDataPoints ? (int) maxDataPoints : argumentLimit; + return new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, endTs, 0, limit, Aggregation.NONE); + } + + private ListenableFuture fetchSingleAggArgumentEntry(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { return switch (argument.getRefEntityKey().getType()) { case TS_ROLLING -> throw new IllegalStateException("TS_ROLLING is not supported for aggregation"); case ATTRIBUTE -> fetchAttributeAggEntry(tenantId, entityId, argument, startTs); @@ -317,36 +341,26 @@ public abstract class AbstractCalculatedFieldProcessingService { }; } - private ListenableFuture> fetchAttributeAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultLastUpdateTs) { + private ListenableFuture fetchAttributeAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultLastUpdateTs) { log.trace("[{}][{}] Fetching attribute for key {}", tenantId, entityId, argument.getRefEntityKey()); var attributeOptFuture = attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()); return Futures.transform(attributeOptFuture, attrOpt -> { log.debug("[{}][{}] Fetched attribute for key {}: {}", tenantId, entityId, argument.getRefEntityKey(), attrOpt); AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, 0L)); - AggSingleArgumentEntry entry = new AggSingleArgumentEntry(entityId, attributeKvEntry); - return Map.entry(entityId, entry); + return new AggSingleArgumentEntry(entityId, attributeKvEntry); }, calculatedFieldCallbackExecutor); } - protected ListenableFuture> fetchTsLatestAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultTs) { + private ListenableFuture fetchTsLatestAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultTs) { String key = argument.getRefEntityKey().getKey(); log.trace("[{}][{}] Fetching latest timeseries {}", tenantId, entityId, key); return Futures.transform( timeseriesService.findLatest(tenantId, entityId, key), result -> { log.debug("[{}][{}] Fetched latest timeseries {}: {}", tenantId, entityId, key, result); - Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))); - AggSingleArgumentEntry entry = new AggSingleArgumentEntry(entityId, tsKvEntry.get()); - return Map.entry(entityId, entry); + Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(defaultTs, createDefaultKvEntry(argument), 0L))); + return new AggSingleArgumentEntry(entityId, tsKvEntry.get()); }, calculatedFieldCallbackExecutor); } - private ReadTsKvQuery buildTsRollingQuery(TenantId tenantId, Argument argument, long startTs, long endTs) { - long maxDataPoints = apiLimitService.getLimit( - tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); - int argumentLimit = argument.getLimit(); - int limit = argumentLimit == 0 || argumentLimit > maxDataPoints ? (int) maxDataPoints : argumentLimit; - return new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, endTs, 0, limit, Aggregation.NONE); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index 4b3e994f23..52b3341151 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -33,7 +33,7 @@ public interface CalculatedFieldProcessingService { ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId); - ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId); + ListenableFuture> fetchAggEntityArguments(CalculatedFieldCtx ctx, EntityId entityId); Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index b7bd4d87fd..f9ed69a313 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -91,8 +91,8 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF } @Override - public ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { - return super.fetchAggArguments(ctx, entityId, System.currentTimeMillis()); + public ListenableFuture> fetchAggEntityArguments(CalculatedFieldCtx ctx, EntityId entityId) { + return super.fetchEntityAggArguments(ctx, entityId, System.currentTimeMillis()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index a75df1fb40..44cb2aaee4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -27,6 +27,7 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -39,6 +40,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationPathLevel; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; @@ -191,19 +193,27 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS for (CalculatedFieldCtx cfCtx : cfCtxs) { EntityId cfEntityId = cfCtx.getEntityId(); if (cfCtx.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - RelationPathLevel relation = aggConfig.getSource().getRelation(); + AggSource source = aggConfig.getSource(); + RelationPathLevel relation = source.getRelation(); EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) ? cfEntityId : calculatedFieldCache.getProfileId(tenantId, cfEntityId); - EntityRelation entityRelation = switch (relation.direction()) { - case FROM -> - relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); - case TO -> - relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0); + switch (relation.direction()) { + case FROM -> { + List byToAndType = relationService.findByToAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); +// List byTo = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); + if (!byToAndType.isEmpty()) { + return true; + } + } + case TO -> { + List byFromAndType = relationService.findByFromAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); +// List byFrom = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); + if (!byFromAndType.isEmpty()) { + return true; + } + } }; - if (entityRelation != null) { - return true; - } } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java index 138053793f..12ae2c4638 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.cf.ctx.state.aggregation; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.script.api.tbel.TbelCfLatestValuesAggregation; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; @@ -66,7 +67,7 @@ public class AggArgumentEntry implements ArgumentEntry { @Override public TbelCfArg toTbelCfArg() { - return null; + return new TbelCfLatestValuesAggregation(aggInputs.values()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json index d402f23c6e..b39092ca74 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json @@ -7,8 +7,8 @@ "allEnabledUntil": 1769907492297 }, "entityId": { - "entityType": "ASSET", - "id": "cc830710-a4cf-11f0-87cb-2d6683c4fccf" + "entityType": "ASSET_PROFILE", + "id": "bb8ddd40-a8bc-11f0-869b-e9d81fa6eaf1" }, "configuration": { "type": "LATEST_VALUES_AGGREGATION", diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 761b69024f..0b5fd3a2c0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -184,7 +184,6 @@ public abstract class AbstractConsumerService { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + + ObjectNode occupancy2 = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy2).isNotNull(); + assertThat(occupancy2.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy2.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + }); + + postTelemetry(device3.getId(), "{\"occupied\":true}"); + + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy2 = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy2).isNotNull(); + assertThat(occupancy2.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy2.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + }); + } + + @Test public void testDeleteRelation_checkMetricsCalculation() throws Exception { deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); @@ -215,6 +260,28 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll }); } +// @Test +// public void testCfWithoutTargetProfileSpecified_checkMetricsCalculation() throws Exception { +// Device device3 = createDevice("Device 3", "1234567890333"); +// postTelemetry(device3.getId(), "{\"occupied\":true}"); +// createEntityRelation(asset.getId(), device3.getId(), "Contains"); +// +// var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) calculatedField.getConfiguration(); +// configuration.getSource().setEntityProfiles(Collections.emptyList()); +// calculatedField.setConfiguration(configuration); +// saveCalculatedField(calculatedField); +// +// await().alias("update cf and perform aggregation for 3 devices").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) +// .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) +// .untilAsserted(() -> { +// ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); +// assertThat(occupancy).isNotNull(); +// assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); +// assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); +// assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); +// }); +// } + private void checkInitialCalculation() { await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) @@ -229,7 +296,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); } - private CalculatedField createOccupancyCF(EntityId entityId, List profiles) { + private CalculatedField createOccupancyCF(String name, EntityId entityId, List profiles) { Map aggMetrics = new HashMap<>(); AggMetric freeSpaces = new AggMetric(); @@ -252,7 +319,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll Output output = new Output(); output.setType(OutputType.TIME_SERIES); - return createAggCf("Occupied spaces", entityId, + return createAggCf(name, entityId, buildSource(EntitySearchDirection.FROM, "Contains", profiles), Map.of("oc", new ReferencedEntityKey("occupied", ArgumentType.TS_LATEST, null)), aggMetrics, diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index 5b3290c110..20348e5922 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; +import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -86,11 +87,17 @@ public interface RelationService { ListenableFuture> findByRelationPathQueryAsync(TenantId tenantId, EntityRelationPathQuery relationPathQuery); + ListenableFuture> findByProfileEntityRelationPathQueryAsync(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery); + + List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery); + + ListenableFuture> findByFromAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId from, String relationType, EntityId targetProfileId); + List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId profileId); - EntityRelation findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId); + ListenableFuture> findByToAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId to, String relationType, EntityId targetProfileId); - void evictRelationsByProfile(TenantId tenantId, EntityId profileId); + List findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId); void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java index 393697877e..65d545bc6e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java @@ -42,7 +42,7 @@ public class CfAggTrigger { } public boolean matchesProfile(EntityId profileId) { - return entityProfiles.contains(profileId); + return entityProfiles.isEmpty() || entityProfiles.contains(profileId); } public boolean matchesTimeSeries(List telemetry) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java index e3db50fda0..2760e1855b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -45,7 +45,7 @@ public class LatestValuesAggregationCalculatedFieldConfiguration implements Calc public CfAggTrigger buildTrigger() { return CfAggTrigger.builder() .inputs(List.copyOf(inputs.values())) - .entityProfiles(source.getEntityProfiles()) + .entityProfiles(List.copyOf(source.getEntityProfiles())) .build(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java new file mode 100644 index 0000000000..32b338ff6f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.relation; + +import org.thingsboard.server.common.data.id.EntityId; + +public record ProfileEntityRelationPathQuery(EntityId rootEntityId, RelationPathLevel level, EntityId targetEntityProfileId) { +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java index 1b5fa394d2..4d1b42aa94 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java @@ -39,6 +39,6 @@ public class TbelCfLatestValuesAggregation implements TbelCfArg { @Override public long memorySize() { - return 32; + return OBJ_SIZE; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index df79ca145a..2d584ebebe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -45,6 +45,7 @@ import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.relation.RelationTypeGroup; @@ -515,32 +516,92 @@ public class BaseRelationService implements RelationService { } @Override - public List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId profileId) { - RelationCacheKey cacheKey = RelationCacheKey.builder().from(from).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.FROM).entityProfile(profileId).build(); - return cache.getAndPutInTransaction(cacheKey, - () -> relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, profileId), - RelationCacheValue::getRelations, - relations -> RelationCacheValue.builder().relations(relations).build(), false); + public ListenableFuture> findByProfileEntityRelationPathQueryAsync(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery) { + log.trace("Executing findByProfileEntityRelationPathQueryAsync, tenantId [{}], relationPathQuery {}", tenantId, relationPathQuery); + validateId(tenantId, id -> "Invalid tenant id: " + id); + validate(relationPathQuery); + RelationPathLevel relationPathLevel = relationPathQuery.level(); + return switch (relationPathLevel.direction()) { + case FROM -> findByFromAndTypeAndEntityProfileAsync(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); + case TO -> findByToAndTypeAndEntityProfileAsync(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); + }; } @Override - public EntityRelation findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId) { - RelationCacheKey cacheKey = RelationCacheKey.builder().to(to).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.TO).entityProfile(profileId).build(); - return cache.getAndPutInTransaction(cacheKey, - () -> relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, profileId), - RelationCacheValue::getRelation, - relation -> RelationCacheValue.builder().relation(relation).build(), false); + public List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery) { + log.trace("Executing findByProfileEntityRelationPathQuery, tenantId [{}], relationPathQuery {}", tenantId, relationPathQuery); + validateId(tenantId, id -> "Invalid tenant id: " + id); + validate(relationPathQuery); + return relationDao.findByProfileEntityRelationPathQuery(tenantId, relationPathQuery); +// RelationPathLevel relationPathLevel = relationPathQuery.level(); +// return switch (relationPathLevel.direction()) { +// case FROM -> findByFromAndTypeAndEntityProfile(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); +// case TO -> findByToAndTypeAndEntityProfile(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); +// }; + } + + @Override + public ListenableFuture> findByFromAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId from, String relationType, EntityId targetProfileId) { + log.trace("Executing findByFromAndTypeAndEntityProfileAsync [{}][{}][{}]", from, relationType, targetProfileId); + validate(from); + validateType(relationType); + if (targetProfileId == null) { + return findByFromAndTypeAsync(tenantId, from, relationType, RelationTypeGroup.COMMON); + } + return executor.submit(() -> findByFromAndTypeAndEntityProfile(tenantId, from, relationType, targetProfileId)); + } + + @Override + public List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId targetProfileId) { + if (targetProfileId == null) { + return findByFromAndType(tenantId, from, relationType, RelationTypeGroup.COMMON); + } +// RelationCacheKey cacheKey = RelationCacheKey.builder().from(from).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.FROM).entityProfile(targetProfileId).build(); +// return cache.getAndPutInTransaction(cacheKey, +// () -> relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, targetProfileId), +// RelationCacheValue::getRelations, +// relations -> RelationCacheValue.builder().relations(relations).build(), false); + + return relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, targetProfileId); } @Override - public void evictRelationsByProfile(TenantId tenantId, EntityId profileId) { - RelationCacheKey key = RelationCacheKey.builder().entityProfile(profileId).build(); - cache.evict(List.of(key)); - log.debug("Processed evict relations by key: {}", key); + public ListenableFuture> findByToAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId to, String relationType, EntityId targetProfileId) { + log.trace("Executing findByToAndTypeAndEntityProfileAsync [{}][{}][{}]", to, relationType, targetProfileId); + validate(to); + validateType(relationType); + if (targetProfileId == null) { + return findByToAndTypeAsync(tenantId, to, relationType, RelationTypeGroup.COMMON); + } + return executor.submit(() -> findByToAndTypeAndEntityProfile(tenantId, to, relationType, targetProfileId)); + } + + @Override + public List findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId targetProfileId) { + if (targetProfileId == null) { + return findByFromAndType(tenantId, to, relationType, RelationTypeGroup.COMMON); + } +// RelationCacheKey cacheKey = RelationCacheKey.builder().to(to).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.TO).entityProfile(targetProfileId).build(); +// return cache.getAndPutInTransaction(cacheKey, +// () -> relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, targetProfileId), +// RelationCacheValue::getRelations, +// relations -> RelationCacheValue.builder().relations(relations).build(), false); + + return relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, targetProfileId); } @Override public void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId) { + +// List keys = new ArrayList<>(5); +// keys.add(new RelationCacheKey(entityId, null, event.getType(), event.getTypeGroup())); +// keys.add(new RelationCacheKey(event.getFrom(), null, event.getType(), event.getTypeGroup(), EntitySearchDirection.FROM)); +// keys.add(new RelationCacheKey(event.getFrom(), null, null, event.getTypeGroup(), EntitySearchDirection.FROM)); +// keys.add(new RelationCacheKey(null, event.getTo(), event.getType(), event.getTypeGroup(), EntitySearchDirection.TO)); +// keys.add(new RelationCacheKey(null, event.getTo(), null, event.getTypeGroup(), EntitySearchDirection.TO)); +// cache.evict(keys); +// log.debug("Processed evict event: {}", event); + List keys = new ArrayList<>(2); keys.add(RelationCacheKey.builder().from(entityId).entityProfile(profileId).build()); keys.add(RelationCacheKey.builder().to(entityId).entityProfile(profileId).build()); @@ -548,6 +609,11 @@ public class BaseRelationService implements RelationService { log.debug("Processed evict relations by keys: {}", keys); } + private void validate(ProfileEntityRelationPathQuery relationPathQuery) { + validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); + relationPathQuery.level().validate(); + } + private void validate(EntityRelationPathQuery relationPathQuery) { validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); List levels = relationPathQuery.levels(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index f529396965..6318f1fc9d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -20,6 +20,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; +import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -40,7 +41,7 @@ public interface RelationDao { List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup); - EntityRelation findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId); + List findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId); List findAllByTo(TenantId tenantId, EntityId to); @@ -78,4 +79,6 @@ public interface RelationDao { List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery relationPathQuery); + List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery query); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index c859b71580..afecca26c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -41,9 +42,12 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; +import static org.thingsboard.server.dao.model.ModelConstants.ASSET_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.DEVICE_TABLE_NAME; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_FROM_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_FROM_TYPE_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_TABLE_NAME; @@ -118,8 +122,14 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple } @Override - public EntityRelation findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { - return DaoUtil.getData(relationRepository.findByToAndProfile(to.getId(), to.getEntityType().name(), typeGroup.name(), relationType, profileId.getId())); + public List findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { + return DaoUtil.convertDataList( + relationRepository.findByToAndProfile( + to.getId(), + to.getEntityType().name(), + typeGroup.name(), + relationType, + profileId.getId())); } @Override @@ -402,4 +412,92 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple return sb.toString(); } + @Override + public List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery query) { + String sql = buildProfileEntityRelationPathSql(query); + Object[] params = buildProfileEntityRelationPathParams(query); + + log.trace("[{}] profile entity relation path query: {}", tenantId, sql); + + return jdbcTemplate.queryForList(sql, params).stream() + .map(row -> { + var entityRelation = new EntityRelation(); + var fromId = (UUID) row.get(RELATION_FROM_ID_PROPERTY); + var fromType = (String) row.get(RELATION_FROM_TYPE_PROPERTY); + var toId = (UUID) row.get(RELATION_TO_ID_PROPERTY); + var toType = (String) row.get(RELATION_TO_TYPE_PROPERTY); + var grp = (String) row.get(RELATION_TYPE_GROUP_PROPERTY); + var type = (String) row.get(RELATION_TYPE_PROPERTY); + var version = (Long) row.get(VERSION_COLUMN); + + entityRelation.setFrom(EntityIdFactory.getByTypeAndUuid(fromType, fromId)); + entityRelation.setTo(EntityIdFactory.getByTypeAndUuid(toType, toId)); + entityRelation.setType(type); + entityRelation.setTypeGroup(RelationTypeGroup.valueOf(grp)); + entityRelation.setVersion(version); + return entityRelation; + }) + .collect(Collectors.toList()); + } + + private Object[] buildProfileEntityRelationPathParams(ProfileEntityRelationPathQuery query) { + final List params = new ArrayList<>(); + + params.add(query.rootEntityId().getId()); + params.add(query.rootEntityId().getEntityType().name()); + + params.add(query.level().relationType()); + + if (query.targetEntityProfileId() != null) { + params.add(query.targetEntityProfileId().getId()); + params.add(query.targetEntityProfileId().getId()); + } + + return params.toArray(); + } + + private static String buildProfileEntityRelationPathSql(ProfileEntityRelationPathQuery query) { + EntitySearchDirection direction = query.level().direction(); + + StringBuilder sb = new StringBuilder(); + + sb.append("\n") + .append("SELECT r.from_id, r.from_type, r.to_id, r.to_type,\n") + .append(" r.relation_type_group, r.relation_type, r.version\n") + .append("FROM ").append(RELATION_TABLE_NAME).append(" r\n"); + + sb.append("JOIN ").append(DEVICE_TABLE_NAME).append(" d ON "); + if (EntitySearchDirection.FROM == direction) { + sb.append("r.to_id = d.id AND r.to_type = 'DEVICE'").append("\n"); + } else { + sb.append("r.from_id = d.id AND r.from_type = 'DEVICE'").append("\n"); + } + + sb.append("JOIN ").append(ASSET_TABLE_NAME).append(" a ON "); + if (EntitySearchDirection.FROM == direction) { + sb.append("r.to_id = a.id AND r.to_type = 'ASSET'").append("\n"); + } else { + sb.append("r.from_id = a.id AND r.from_type = 'ASSET'").append("\n"); + } + + if (EntitySearchDirection.FROM == direction) { + sb.append("WHERE r.from_id = ?").append("\n") + .append("AND r.from_type = ?").append("\n"); + } else { + sb.append("WHERE r.to_id = ?").append("\n") + .append("AND r.to_type = ?").append("\n"); + } + + sb.append("AND r.relation_type = ?").append("\n") + .append("AND r.relation_type_group = '").append(RelationTypeGroup.COMMON).append("'\n"); + + if (query.targetEntityProfileId() != null) { + sb.append("AND ((d.device_profile_id = ?) OR (a.asset_profile_id = ?))").append("\n"); + } + + sb.append("AND (d.id IS NOT NULL OR a.id IS NOT NULL)"); + + return sb.toString(); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index 9294236526..0ebd5b6ceb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -126,9 +126,8 @@ public interface RelationRepository AND r.relation_type_group = :relationTypeGroup AND ((d.device_profile_id = :profileId) OR (a.asset_profile_id = :profileId)) AND (d.id IS NOT NULL OR a.id IS NOT NULL) - LIMIT 1 """, nativeQuery = true) - Optional findByToAndProfile(@Param("toId") UUID toId, + List findByToAndProfile(@Param("toId") UUID toId, @Param("toType") String toType, @Param("relationTypeGroup") String relationTypeGroup, @Param("relationType") String relationType, From 9ac6219882550ee6b138f6d10807096d313e7eda Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 14 Oct 2025 17:05:47 +0300 Subject: [PATCH 03/33] removed entityProfiles from config --- ...CalculatedFieldEntityMessageProcessor.java | 33 ++-- ...alculatedFieldManagerMessageProcessor.java | 143 ++++----------- ...tractCalculatedFieldProcessingService.java | 65 +++---- .../service/cf/CalculatedFieldCache.java | 3 +- .../cf/DefaultCalculatedFieldCache.java | 26 ++- .../DefaultCalculatedFieldQueueService.java | 31 +--- .../service/cf/ctx/state/ArgumentEntry.java | 4 + .../cf/ctx/state/CalculatedFieldCtx.java | 112 ++++-------- .../state/aggregation/function/new_agg.json | 60 +++++++ .../utils/CalculatedFieldArgumentUtils.java | 9 + ...tValuesAggregationCalculatedFieldTest.java | 167 ++++++++++-------- .../aggregation/CfAggTrigger.java | 104 ----------- ...gregationCalculatedFieldConfiguration.java | 19 +- 13 files changed, 297 insertions(+), 479 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 9dfde4d821..e253633727 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -171,8 +171,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM throw cfe; } throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); - } - } + } } public void process(CalculatedFieldArgumentResetMsg msg) throws CalculatedFieldException { log.debug("[{}] Processing CF argument reset msg.", entityId); @@ -523,22 +522,21 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { - return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getAggregationInputs(), data); + return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getRelatedEntityArguments(), data); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { - return mapToArguments(entityId, ctx.getLinkedAndDynamicArgs(entityId), ctx.getAggregationInputs(), data); + return mapToArguments(entityId, ctx.getLinkedAndDynamicArgs(entityId), ctx.getRelatedEntityArguments(), data); } - private Map mapToArguments(EntityId originator, Map argNames, Map aggArgNames, List data) { + private Map mapToArguments(EntityId originator, Map argNames, Map aggArgNames, List data) { Map arguments = new HashMap<>(); if (!aggArgNames.isEmpty()) { - for (Map.Entry entry : aggArgNames.entrySet()) { - for (TsKvProto item : data) { - ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); - if (key.equals(entry.getValue())) { - arguments.put(entry.getKey(), new AggSingleArgumentEntry(originator, item)); - } + for (TsKvProto item : data) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); + String argName = aggArgNames.get(key); + if (argName != null) { + arguments.put(argName, new AggSingleArgumentEntry(originator, item)); } } } @@ -560,17 +558,17 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List attrDataList) { - return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), ctx.getAggregationInputs(), scope, attrDataList); + return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), ctx.getRelatedEntityArguments(), scope, attrDataList); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { var argNames = ctx.getLinkedAndDynamicArgs(entityId); List geofencingArgumentNames = ctx.getLinkedEntityAndCurrentOwnerGeofencingArgumentNames(); - Map aggregationInputs = ctx.getAggregationInputs(); + Map aggregationInputs = ctx.getRelatedEntityArguments(); return mapToArguments(entityId, argNames, geofencingArgumentNames, aggregationInputs, scope, attrDataList); } - private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, Map aggArgNames, AttributeScopeProto scope, List attrDataList) { + private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, Map aggArgNames, AttributeScopeProto scope, List attrDataList) { Map arguments = new HashMap<>(); if (!argNames.isEmpty()) { for (AttributeValueProto item : attrDataList) { @@ -589,10 +587,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (!aggArgNames.isEmpty()) { for (AttributeValueProto item : attrDataList) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); - for (Map.Entry entry : aggArgNames.entrySet()) { - if (key.equals(entry.getValue())) { - arguments.put(entry.getKey(), new AggSingleArgumentEntry(entityId, item)); - } + String argName = aggArgNames.get(key); + if (argName != null) { + arguments.put(argName, new AggSingleArgumentEntry(entityId, item)); } } } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 3c1e21bcc8..620eee41ee 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -24,7 +24,6 @@ import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.calculatedField.EntityInitCalculatedFieldMsg.StateAction; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; @@ -32,20 +31,17 @@ import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; -import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationPathLevel; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.msg.CalculatedFieldStatePartitionRestoreMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; @@ -53,13 +49,11 @@ import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldStateService; @@ -76,8 +70,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ScheduledFuture; @@ -96,7 +88,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final Map> entityIdCalculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFieldLinks = new HashMap<>(); private final Map> ownerEntities = new HashMap<>(); - private final Map cfTriggers = new HashMap<>(); + private final Map aggCalculatedFields = new HashMap<>(); private ScheduledFuture cfsReevaluationTask; private final CalculatedFieldProcessingService cfExecService; @@ -146,7 +138,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsReevaluationTask.cancel(true); cfsReevaluationTask = null; } - cfTriggers.clear(); + aggCalculatedFields.clear(); ctx.stop(ctx.getSelf()); } @@ -252,19 +244,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } updateEntityOwner(entityId); - MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); - - // process aggregation cfs(in any) - List cfsRelatedToEntity = getCfsWithRelationToEntity(entityId, profileId); - if (!cfsRelatedToEntity.isEmpty()) { - MultipleTbCallback multiCallback = new MultipleTbCallback(cfsRelatedToEntity.size(), callbackFor2); - cfsRelatedToEntity.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> initRelatedEntity(id, entityId, ctx, cb)); - }); - } else { - callbackFor2.onSuccess(); - } - if (!isMyPartition(entityId, callback)) { return; } @@ -272,11 +251,11 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var profileIdFields = getCalculatedFieldsByEntityId(profileId); var fieldsCount = entityIdFields.size() + profileIdFields.size(); if (fieldsCount > 0) { - MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callbackFor2); + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); entityIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); profileIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); } else { - callbackFor2.onSuccess(); + callback.onSuccess(); } } @@ -286,35 +265,16 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (!isMyPartition(msg.getEntityId(), callback)) { return; } - MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); - - // process aggregation cfs(in any) - List oldCfsRelatedToEntity = getCfsWithRelationToEntity(msg.getEntityId(), msg.getOldProfileId()); - List newCfsRelatedToEntity = getCfsWithRelationToEntity(msg.getEntityId(), msg.getProfileId()); - var fieldsWithRelatedEntityCount = oldCfsRelatedToEntity.size() + newCfsRelatedToEntity.size(); - if (fieldsWithRelatedEntityCount > 0) { - MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsWithRelatedEntityCount, callbackFor2); - var entityId = msg.getEntityId(); - oldCfsRelatedToEntity.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> deleteRelatedEntity(id, entityId, cb)); - }); - newCfsRelatedToEntity.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> initRelatedEntity(id, entityId, ctx, cb)); - }); - } else { - callbackFor2.onSuccess(); - } - var oldProfileCfs = getCalculatedFieldsByEntityId(msg.getOldProfileId()); var newProfileCfs = getCalculatedFieldsByEntityId(msg.getProfileId()); var fieldsCount = oldProfileCfs.size() + newProfileCfs.size(); if (fieldsCount > 0) { - MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callbackFor2); + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); var entityId = msg.getEntityId(); oldProfileCfs.forEach(ctx -> deleteCfForEntity(entityId, ctx.getCfId(), multiCallback)); newProfileCfs.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); } else { - callbackFor2.onSuccess(); + callback.onSuccess(); } } else if (msg.isOwnerChanged()) { onEntityOwnerChanged(msg, callback); @@ -334,10 +294,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware case CUSTOMER -> ownerEntities.remove(msg.getEntityId()); } ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); - - getCfsWithRelationToEntity(msg.getEntityId(), msg.getProfileId()).forEach(ctx -> { - applyToTargetCfEntityActors(ctx, callback, (id, cb) -> deleteRelatedEntity(id, msg.getEntityId(), cb)); - }); if (isMyPartition(msg.getEntityId(), callback)) { log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); @@ -365,9 +321,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsByToIdOrItsProfileId.forEach(cf -> { var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - AggSource source = configuration.getSource(); - RelationPathLevel relation = source.getRelation(); - if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(fromIdProfile)) { + RelationPathLevel relation = configuration.getRelation(); + if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType())) { toIdMatches.add(cf); } }); @@ -386,9 +341,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsByFromIdOrItsProfileId.forEach(cf -> { var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - AggSource source = configuration.getSource(); - RelationPathLevel relation = source.getRelation(); - if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(toIdProfile)) { + RelationPathLevel relation = configuration.getRelation(); + if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType())) { fromIdMatches.add(cf); } }); @@ -424,9 +378,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsByToIdOrItsProfileId.forEach(cf -> { var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - AggSource source = configuration.getSource(); - RelationPathLevel relation = source.getRelation(); - if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(fromIdProfile)) { + RelationPathLevel relation = configuration.getRelation(); + if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType())) { toIdMatches.add(cf); } }); @@ -445,9 +398,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsByFromIdOrItsProfileId.forEach(cf -> { var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - AggSource source = configuration.getSource(); - RelationPathLevel relation = source.getRelation(); - if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(toIdProfile)) { + RelationPathLevel relation = configuration.getRelation(); + if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType())) { fromIdMatches.add(cf); } }); @@ -482,7 +434,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } calculatedFields.put(cf.getId(), cfCtx); if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + aggCalculatedFields.put(cf.getId(), cfCtx); } // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) @@ -515,8 +467,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(newCfCtx).eventEntity(newCfCtx.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(newCf.getId(), newCfCtx); - if (newCf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - cfTriggers.put(newCf.getId(), aggConfig.buildTrigger()); + if (newCf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + aggCalculatedFields.put(newCf.getId(), newCfCtx); } List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new CopyOnWriteArrayList<>(); @@ -559,7 +511,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onCfDeleted(ComponentLifecycleMsg msg, TbCallback callback) { var cfId = new CalculatedFieldId(msg.getEntityId().getId()); var cfCtx = calculatedFields.remove(cfId); // fixme wtf? why isn't ctx closed properly? - cfTriggers.remove(cfId); + aggCalculatedFields.remove(cfId); if (cfCtx == null) { log.debug("[{}] CF was already deleted [{}]", tenantId, cfId); callback.onSuccess(); @@ -614,21 +566,14 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private List filterAggregationCfs(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); - return cfTriggers.entrySet().stream() - .filter(entry -> aggMatches(entry.getValue(), msg.getProto())) - .map(Entry::getKey) - .map(calculatedFields::get) - .filter(Objects::nonNull) + return aggCalculatedFields.values().stream() + .filter(cf -> cf.relatedEntityMatches(msg.getProto())) .flatMap(cf -> findRelationsForCf(entityId, cf).stream()) .toList(); } - private List getCfsWithRelationToEntity(EntityId entityId, EntityId profileId) { - return cfTriggers.entrySet().stream() - .filter(entry -> entry.getValue().matchesProfile(profileId)) - .map(Entry::getKey) - .map(calculatedFields::get) - .filter(Objects::nonNull) + private List getCfsWithRelationToEntity(EntityId entityId) { + return aggCalculatedFields.values().stream() .filter(cf -> !findRelationsForCf(entityId, cf).isEmpty()) .toList(); } @@ -636,24 +581,19 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private List findRelationsForCf(EntityId entityId, CalculatedFieldCtx cf) { List result = new ArrayList<>(); if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration configuration) { - AggSource source = configuration.getSource(); - RelationPathLevel relation = source.getRelation(); - EntityId cfEntityId = cf.getEntityId(); - EntityId targetProfileId = isProfileEntity(cfEntityId.getEntityType()) - ? cfEntityId - : getProfileId(tenantId, cfEntityId); + RelationPathLevel relation = configuration.getRelation(); switch (relation.direction()) { case FROM -> { - List relationsByTo = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), targetProfileId); - if (relationsByTo != null && !relationsByTo.isEmpty()) { - EntityRelation entityRelation = relationsByTo.get(0); // only one supported + List byToAndType = relationService.findByToAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); + if (byToAndType != null && !byToAndType.isEmpty()) { + EntityRelation entityRelation = byToAndType.get(0); // only one supported result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getFrom())); } } case TO -> { - List relationsByFrom = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), targetProfileId); - if (relationsByFrom != null && !relationsByFrom.isEmpty()) { - for (EntityRelation entityRelation : relationsByFrom) { + List byFromAndType = relationService.findByFromAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); + if (byFromAndType != null && !byFromAndType.isEmpty()) { + for (EntityRelation entityRelation : byFromAndType) { if (entityRelation.getTo().equals(cf.getEntityId())) { result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getTo())); } @@ -665,25 +605,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return result; } - private boolean aggMatches(CfAggTrigger cfAggTrigger, CalculatedFieldTelemetryMsgProto proto) { - if (!proto.getTsDataList().isEmpty()) { - List updatedTelemetry = proto.getTsDataList().stream() - .map(ProtoUtils::fromProto) - .toList(); - return cfAggTrigger.matchesTimeSeries(updatedTelemetry); - } else if (!proto.getAttrDataList().isEmpty()) { - AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); - List updatedTelemetry = proto.getAttrDataList().stream() - .map(ProtoUtils::fromProto) - .toList(); - return cfAggTrigger.matchesAttributes(updatedTelemetry, scope); - } else if (!proto.getRemovedTsKeysList().isEmpty()) { - return cfAggTrigger.matchesTimeSeriesKeys(proto.getRemovedTsKeysList()); - } else { - return cfAggTrigger.matchesAttributesKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); - } - } - public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { EntityId sourceEntityId = msg.getEntityId(); log.debug("Received linked telemetry msg from entity [{}]", sourceEntityId); @@ -889,8 +810,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(cf.getId(), cfCtx); - if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + aggCalculatedFields.put(cf.getId(), cfCtx); } // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index db15fc1dc8..c30a12a8f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -27,7 +27,6 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; -import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -39,7 +38,7 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; +import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.attributes.AttributesService; @@ -49,13 +48,11 @@ import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -65,6 +62,7 @@ import static org.thingsboard.server.common.data.cf.configuration.geofencing.Ent import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LONGITUDE_ARGUMENT_KEY; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultAttributeEntry; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultKvEntry; +import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformAggSingleArgument; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformSingleValueArgument; @Data @@ -123,30 +121,19 @@ public abstract class AbstractCalculatedFieldProcessingService { return resolveOwnerArgument(tenantId, entityId); } - private ListenableFuture> resolveRelatedEntities(TenantId tenantId, EntityId entityId, AggSource aggSource) { - RelationPathLevel relation = aggSource.getRelation(); + private ListenableFuture> resolveRelatedEntities(TenantId tenantId, EntityId entityId, RelationPathLevel relation) { + ListenableFuture> relationsFut = relationService.findByRelationPathQueryAsync(tenantId, new EntityRelationPathQuery(entityId, List.of(relation))); - List>> relationListsFut = new ArrayList<>(); - if (aggSource.getEntityProfiles().isEmpty()) { - relationListsFut.add(relationService.findByProfileEntityRelationPathQueryAsync(tenantId, new ProfileEntityRelationPathQuery(entityId, relation, null))); - } else { - aggSource.getEntityProfiles().forEach(profile -> relationListsFut.add(relationService.findByProfileEntityRelationPathQueryAsync(tenantId, new ProfileEntityRelationPathQuery(entityId, relation, profile)))); - } - - return Futures.transform(Futures.allAsList(relationListsFut), relationLists -> { - if (relationLists == null) { + return Futures.transform(relationsFut, relations -> { + if (relations == null) { return new ArrayList<>(); } - List allRelations = relationLists.stream() - .filter(Objects::nonNull) - .flatMap(List::stream) - .toList(); return switch (relation.direction()) { - case FROM -> allRelations.stream() + case FROM -> relations.stream() .map(EntityRelation::getTo) .toList(); - case TO -> allRelations.isEmpty() ? List.of() : List.of(allRelations.get(0).getFrom()); + case TO -> relations.isEmpty() ? List.of() : List.of(relations.get(0).getFrom()); }; }, calculatedFieldCallbackExecutor); } @@ -193,28 +180,26 @@ public abstract class AbstractCalculatedFieldProcessingService { protected Map> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - ListenableFuture> relatedEntities = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getSource()); + ListenableFuture> relatedEntitiesFut = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getRelation()); - Map> futures = new HashMap<>(); - aggConfig.getInputs().forEach((key, refKey) -> { - Argument argument = new Argument(); - argument.setRefEntityKey(refKey); - futures.put(key, Futures.transformAsync(relatedEntities, entityIds -> fetchAggArgumentEntry(ctx.getTenantId(), entityIds, argument, System.currentTimeMillis()), MoreExecutors.directExecutor())); - }); - return futures; + return aggConfig.getArguments().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> Futures.transformAsync(relatedEntitiesFut, relatedEntities -> fetchAggArgumentEntry(ctx.getTenantId(), relatedEntities, entry.getValue(), ts), MoreExecutors.directExecutor()) + )); } protected ListenableFuture> fetchEntityAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - Map> futures = new HashMap<>(); - aggConfig.getInputs().forEach((key, refKey) -> { - Argument argument = new Argument(); - argument.setRefEntityKey(refKey); - ListenableFuture argEntryFut = fetchSingleAggArgumentEntry(ctx.getTenantId(), entityId, argument, ts); - futures.put(key, argEntryFut); - }); - return Futures.whenAllComplete(futures.values()) - .call(() -> resolveArgumentFutures(futures), + + Map> argsFutures = aggConfig.getArguments().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> fetchSingleAggArgumentEntry(ctx.getTenantId(), entityId, entry.getValue(), ts) + )); + + return Futures.whenAllComplete(argsFutures.values()) + .call(() -> resolveArgumentFutures(argsFutures), MoreExecutors.directExecutor()); } @@ -347,7 +332,7 @@ public abstract class AbstractCalculatedFieldProcessingService { return Futures.transform(attributeOptFuture, attrOpt -> { log.debug("[{}][{}] Fetched attribute for key {}: {}", tenantId, entityId, argument.getRefEntityKey(), attrOpt); AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, 0L)); - return new AggSingleArgumentEntry(entityId, attributeKvEntry); + return transformAggSingleArgument(entityId, Optional.of(attributeKvEntry)); }, calculatedFieldCallbackExecutor); } @@ -359,7 +344,7 @@ public abstract class AbstractCalculatedFieldProcessingService { result -> { log.debug("[{}][{}] Fetched latest timeseries {}: {}", tenantId, entityId, key, result); Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(defaultTs, createDefaultKvEntry(argument), 0L))); - return new AggSingleArgumentEntry(entityId, tsKvEntry.get()); + return transformAggSingleArgument(entityId, tsKvEntry); }, calculatedFieldCallbackExecutor); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index e32ca42f9c..27e989de70 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -39,7 +38,7 @@ public interface CalculatedFieldCache { List getCalculatedFieldCtxsByEntityId(EntityId entityId); - List getCalculatedFieldCtxsByTrigger(EntityId profileId, Predicate cfAggFilter); + List getAggCalculatedFieldCtxsByFilter(Predicate relatedEntityFilter); boolean hasCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index ae8238fb8a..918d71e326 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -43,8 +42,6 @@ import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -52,7 +49,6 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Predicate; -import java.util.stream.Collectors; @Service @Slf4j @@ -73,7 +69,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); - private final ConcurrentMap cfTriggers = new ConcurrentHashMap<>(); + private final ConcurrentMap aggCalculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> ownerEntities = new ConcurrentHashMap<>(); @@ -87,8 +83,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { cfs.forEach(cf -> { if (cf != null) { calculatedFields.putIfAbsent(cf.getId(), cf); - if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + aggCalculatedFields.put(cf.getId(), cf); } } }); @@ -156,13 +152,11 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } @Override - public List getCalculatedFieldCtxsByTrigger(EntityId profileId, Predicate cfAggFilter) { - return cfTriggers.entrySet().stream() - .filter(entry -> entry.getValue().matches(profileId, cfAggFilter)) - .map(Map.Entry::getKey) + public List getAggCalculatedFieldCtxsByFilter(Predicate relatedEntityFilter) { + return aggCalculatedFields.keySet().stream() .map(this::getCalculatedFieldCtx) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + .filter(relatedEntityFilter) + .toList(); } @Override @@ -206,8 +200,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); - if (configuration instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - cfTriggers.put(calculatedField.getId(), aggConfig.buildTrigger()); + if (configuration instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + aggCalculatedFields.put(calculatedField.getId(), calculatedField); } calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); @@ -240,7 +234,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); entityIdCalculatedFieldLinks.forEach((entityId, calculatedFieldLinks) -> calculatedFieldLinks.removeIf(link -> link.getCalculatedFieldId().equals(calculatedFieldId))); log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); - cfTriggers.remove(calculatedFieldId); + aggCalculatedFields.remove(calculatedFieldId); log.debug("[{}] evict calculated field from cached triggers: {}", calculatedFieldId, oldCalculatedField); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 44cb2aaee4..37c7b2ef6c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -27,8 +27,6 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; -import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -89,7 +87,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matches(entries), cf -> cf.linkMatches(entityId, entries), cf -> cf.dynamicSourceMatches(request.getEntries()), - cfTrigger -> cfTrigger.matchesTimeSeries(entries), + cf -> cf.relatedEntityMatches(entries), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -108,7 +106,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matches(entries, scope), cf -> cf.linkMatches(entityId, entries, scope), cf -> cf.dynamicSourceMatches(request.getEntries(), request.getScope()), - cfTrigger -> cfTrigger.matchesAttributes(entries, scope), + cf -> cf.relatedEntityMatches(entries, scope), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -126,7 +124,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matchesKeys(result, scope), cf -> cf.linkMatchesAttrKeys(entityId, result, scope), cf -> cf.matchesDynamicSourceKeys(result, request.getScope()), - cfTrigger -> cfTrigger.matchesAttributesKeys(result, scope), + cf -> cf.matchesRelatedEntityKeys(result, scope), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -138,7 +136,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matchesKeys(result), cf -> cf.linkMatchesTsKeys(entityId, result), cf -> cf.matchesDynamicSourceKeys(result), - cfTrigger -> cfTrigger.matchesTimeSeriesKeys(result), + cf -> cf.matchesRelatedEntityKeys(result), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -146,12 +144,12 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS Predicate mainEntityFilter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter, - Predicate cfAggKeysFilter, + Predicate relatedEntityFilter, Supplier msg, FutureCallback callback) { if (EntityType.TENANT.equals(entityId.getEntityType())) { tenantId = (TenantId) entityId; } - boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter, dynamicSourceFilter, cfAggKeysFilter); + boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter, dynamicSourceFilter, relatedEntityFilter); if (send) { ToCalculatedFieldMsg calculatedFieldMsg = msg.get(); clusterService.pushMsgToCalculatedFields(tenantId, entityId, calculatedFieldMsg, wrap(callback)); @@ -162,7 +160,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } } - private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter, Predicate cfAggKeysFilter) { + private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter, Predicate relatedEntityFilter) { if (!CalculatedField.SUPPORTED_REFERENCED_ENTITIES.contains(entityId.getEntityType())) { return false; } @@ -189,26 +187,19 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } } - List cfCtxs = calculatedFieldCache.getCalculatedFieldCtxsByTrigger(calculatedFieldCache.getProfileId(tenantId, entityId), cfAggKeysFilter); + List cfCtxs = calculatedFieldCache.getAggCalculatedFieldCtxsByFilter(relatedEntityFilter); for (CalculatedFieldCtx cfCtx : cfCtxs) { - EntityId cfEntityId = cfCtx.getEntityId(); if (cfCtx.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - AggSource source = aggConfig.getSource(); - RelationPathLevel relation = source.getRelation(); - EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) - ? cfEntityId - : calculatedFieldCache.getProfileId(tenantId, cfEntityId); + RelationPathLevel relation = aggConfig.getRelation(); switch (relation.direction()) { case FROM -> { List byToAndType = relationService.findByToAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); -// List byTo = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); if (!byToAndType.isEmpty()) { return true; } } case TO -> { List byFromAndType = relationService.findByFromAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); -// List byFrom = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); if (!byFromAndType.isEmpty()) { return true; } @@ -307,10 +298,6 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS return telemetryMsg; } - private boolean isProfileEntity(EntityType entityType) { - return EntityType.DEVICE_PROFILE.equals(entityType) || EntityType.ASSET_PROFILE.equals(entityType); - } - private static TbQueueCallback wrap(FutureCallback callback) { if (callback != null) { return new FutureCallbackWrapper(callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index f22b10a9c1..4e3d00ee62 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -74,4 +74,8 @@ public interface ArgumentEntry { return new AggArgumentEntry(entityIdkvEntryMap, false); } + static ArgumentEntry createAggSingleArgument(EntityId entityId, KvEntry kvEntry) { + return new AggSingleArgumentEntry(entityId, kvEntry); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 963ec3f06f..f8b9f5b665 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -83,8 +83,7 @@ public class CalculatedFieldCtx { private final Map mainEntityArguments; private final Map> linkedEntityArguments; private final Map dynamicEntityArguments; - private final List aggInputs; - private final Map aggregationInputs; + private final Map relatedEntityArguments; private final List argNames; private Output output; private String expression; @@ -123,11 +122,10 @@ public class CalculatedFieldCtx { this.mainEntityArguments = new HashMap<>(); this.linkedEntityArguments = new HashMap<>(); this.dynamicEntityArguments = new HashMap<>(); + this.relatedEntityArguments = new HashMap<>(); this.argNames = new ArrayList<>(); this.mainEntityGeofencingArgumentNames = new ArrayList<>(); this.linkedEntityAndCurrentOwnerGeofencingArgumentNames = new ArrayList<>(); - this.aggInputs = new ArrayList<>(); - this.aggregationInputs = new HashMap<>(); this.output = calculatedField.getConfiguration().getOutput(); if (calculatedField.getConfiguration() instanceof ArgumentsBasedCalculatedFieldConfiguration argBasedConfig) { this.arguments.putAll(argBasedConfig.getArguments()); @@ -135,6 +133,10 @@ public class CalculatedFieldCtx { var refId = entry.getValue().getRefEntityId(); var refKey = entry.getValue().getRefEntityKey(); if (refId == null) { + if (CalculatedFieldType.LATEST_VALUES_AGGREGATION.equals(cfType)) { + relatedEntityArguments.put(refKey, entry.getKey()); + continue; + } if (entry.getValue().hasRelationQuerySource()) { relationQueryDynamicArguments = true; continue; @@ -172,9 +174,6 @@ public class CalculatedFieldCtx { } this.requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - aggInputs.addAll(aggConfig.getInputs().values()); - aggregationInputs.putAll(aggConfig.getInputs()); - this.argNames.addAll(aggConfig.getInputs().keySet()); this.scheduledUpdateIntervalMillis = aggConfig.getDeduplicationIntervalMillis(); } this.systemContext = systemContext; @@ -475,41 +474,57 @@ public class CalculatedFieldCtx { return map != null && matchesTimeSeriesKeys(map, keys); } - public boolean dynamicSourceMatches(CalculatedFieldTelemetryMsgProto proto) { + public boolean relatedEntityMatches(List values) { + return matchesTimeSeries(relatedEntityArguments, values); + } + + public boolean relatedEntityMatches(List values, AttributeScope scope) { + return matchesAttributes(relatedEntityArguments, values, scope); + } + + public boolean matchesRelatedEntityKeys(List keys, AttributeScope scope) { + return matchesAttributesKeys(relatedEntityArguments, keys, scope); + } + + public boolean matchesRelatedEntityKeys(List keys) { + return matchesTimeSeriesKeys(relatedEntityArguments, keys); + } + + public boolean relatedEntityMatches(CalculatedFieldTelemetryMsgProto proto) { if (!proto.getTsDataList().isEmpty()) { List updatedTelemetry = proto.getTsDataList().stream() .map(ProtoUtils::fromProto) .toList(); - return dynamicSourceMatches(updatedTelemetry); + return relatedEntityMatches(updatedTelemetry); } else if (!proto.getAttrDataList().isEmpty()) { AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); List updatedTelemetry = proto.getAttrDataList().stream() .map(ProtoUtils::fromProto) .toList(); - return dynamicSourceMatches(updatedTelemetry, scope); + return relatedEntityMatches(updatedTelemetry, scope); } else if (!proto.getRemovedTsKeysList().isEmpty()) { - return matchesDynamicSourceKeys(proto.getRemovedTsKeysList()); + return matchesRelatedEntityKeys(proto.getRemovedTsKeysList()); } else { - return matchesDynamicSourceKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); + return matchesRelatedEntityKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); } } - public boolean aggMatches(CalculatedFieldTelemetryMsgProto proto) { + public boolean dynamicSourceMatches(CalculatedFieldTelemetryMsgProto proto) { if (!proto.getTsDataList().isEmpty()) { List updatedTelemetry = proto.getTsDataList().stream() .map(ProtoUtils::fromProto) .toList(); - return matchesAggTimeSeries(updatedTelemetry); + return dynamicSourceMatches(updatedTelemetry); } else if (!proto.getAttrDataList().isEmpty()) { AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); List updatedTelemetry = proto.getAttrDataList().stream() .map(ProtoUtils::fromProto) .toList(); - return matchesAggAttributes(updatedTelemetry, scope); + return dynamicSourceMatches(updatedTelemetry, scope); } else if (!proto.getRemovedTsKeysList().isEmpty()) { - return matchesAggKeys(proto.getRemovedTsKeysList()); + return matchesDynamicSourceKeys(proto.getRemovedTsKeysList()); } else { - return matchesAggAttributesKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); + return matchesDynamicSourceKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); } } @@ -544,67 +559,6 @@ public class CalculatedFieldCtx { return argNames; } - public boolean matchesAggKeys(List values) { - if (aggInputs.isEmpty() || values.isEmpty()) { - return false; - } - - for (String key : values) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); - if (aggInputs.contains(latestKey)) { - return true; - } - } - - return false; - } - - public boolean matchesAggTimeSeries(List values) { - if (aggInputs.isEmpty() || values.isEmpty()) { - return false; - } - - for (TsKvEntry tsKvEntry : values) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKvEntry.getKey(), ArgumentType.TS_LATEST, null); - if (aggInputs.contains(latestKey)) { - return true; - } - } - - return false; - } - - public boolean matchesAggAttributesKeys(List keys, AttributeScope scope) { - if (keys == null || keys.isEmpty()) { - return false; - } - - for (String key : keys) { - ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); - if (aggInputs.contains(attrKey)) { - return true; - } - } - - return false; - } - - public boolean matchesAggAttributes(List keys, AttributeScope scope) { - if (keys == null || keys.isEmpty()) { - return false; - } - - for (AttributeKvEntry attributeKvEntry : keys) { - ReferencedEntityKey attrKey = new ReferencedEntityKey(attributeKvEntry.getKey(), ArgumentType.ATTRIBUTE, scope); - if (aggInputs.contains(attrKey)) { - return true; - } - } - - return false; - } - - public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); } @@ -662,7 +616,7 @@ public class CalculatedFieldCtx { private boolean hasLatestValuesAggregationConfigurationChanges(CalculatedFieldCtx other) { if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig && other.calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig) { - return !thisConfig.getInputs().equals(otherConfig.getInputs()) || !thisConfig.getSource().equals(otherConfig.getSource()); + return !thisConfig.getArguments().equals(otherConfig.getArguments()) || !thisConfig.getRelation().equals(otherConfig.getRelation()); } return false; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json new file mode 100644 index 0000000000..32cd053d41 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json @@ -0,0 +1,60 @@ +{ + "type": "LATEST_VALUES_AGGREGATION", + "name": "Occupied spaces", + "debugSettings": { + "failuresEnabled": true, + "allEnabled": true, + "allEnabledUntil": 1769907492297 + }, + "entityId": { + "entityType": "ASSET_PROFILE", + "id": "2b759c60-a8f4-11f0-be29-7fa922118588" + }, + "configuration": { + "type": "LATEST_VALUES_AGGREGATION", + "relation": { + "direction": "FROM", + "relationType": "Contains" + }, + "arguments": { + "oc": { + "refEntityKey": { + "key": "occupied", + "type": "TS_LATEST" + }, + "defaultValue": "false" + } + }, + "deduplicationIntervalMillis": 20000, + "metrics": { + "totalSpaces": { + "function": "COUNT", + "input": { + "type": "function", + "function" : "return 1;" + } + }, + "occupiedSpaces": { + "function": "COUNT", + "filter": "return oc == true", + "input": { + "type": "key", + "key" : "oc" + } + }, + "freeSpaces": { + "function": "COUNT", + "filter": "return oc == false", + "input": { + "type": "key", + "key" : "oc" + } + } + }, + "output": { + "type": "TIME_SERIES", + "decimals": 2 + }, + "useLatestTsFromInputs": "true" + } +} diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index 6f6fabbea2..6485508602 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -34,6 +34,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -54,6 +55,14 @@ public class CalculatedFieldArgumentUtils { } } + public static ArgumentEntry transformAggSingleArgument(EntityId entityId, Optional kvEntry) { + if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { + return ArgumentEntry.createAggSingleArgument(entityId, kvEntry.get()); + } else { + return new AggSingleArgumentEntry(); + } + } + public static KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getRefEntityKey().getKey(); String defaultValue = argument.getDefaultValue(); diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java index 2ed3554666..cd9c1c578b 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -27,8 +27,8 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; @@ -36,7 +36,6 @@ import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFuncti import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; -import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; @@ -53,9 +52,7 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.service.DaoSqlTest; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -77,8 +74,6 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll private AssetProfile assetProfile; private Asset asset; - private CalculatedField calculatedField; - private long deduplicationInterval = 10000; @Before @@ -111,10 +106,6 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset.getId(), device1.getId(), "Contains"); createEntityRelation(asset.getId(), device2.getId(), "Contains"); - - calculatedField = createOccupancyCF("Occupied spaces", asset.getId(), List.of(deviceProfile.getId())); - - checkInitialCalculation(); } @After @@ -124,8 +115,33 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll deleteTenant(savedTenant.getId()); } + @Test + public void testNoTelemetryOnDevices_checkDefaultValueUsed() throws Exception { + Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + Device device3 = createDevice("Device 3", "1234567890333"); + Device device4 = createDevice("Device 4", "1234567890444"); + + createEntityRelation(asset2.getId(), device3.getId(), "Contains"); + createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + + createOccupancyCF("Occupied spaces", asset2.getId()); + + await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + }); + } + @Test public void testUpdateTelemetry_checkMetricsCalculation() throws Exception { + createOccupancyCF("Occupied spaces", asset.getId()); + checkInitialCalculation(); + postTelemetry(device1.getId(), "{\"occupied\":false}"); await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) @@ -141,6 +157,9 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll @Test public void testUpdateTelemetry_checkMetricsCalculationNotExecutedUntilDeduplicationInterval() throws Exception { + createOccupancyCF("Occupied spaces", asset.getId()); + checkInitialCalculation(); + postTelemetry(device1.getId(), "{\"occupied\":false}"); await().alias("update telemetry -> no changes").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) @@ -161,35 +180,33 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll } @Test - public void testChangeProfile_checkMetricsCalculation() throws Exception { - DeviceProfile deviceProfile2 = doPost("/api/deviceProfile", createDeviceProfile("Device Profile 2"), DeviceProfile.class); - device1.setDeviceProfileId(deviceProfile2.getId()); - device1 = doPost("/api/device?accessToken=" + accessToken1, device1, Device.class); + public void testCreateRelation_checkMetricsCalculation() throws Exception { + createOccupancyCF("Occupied spaces", asset.getId()); + checkInitialCalculation(); - postTelemetry(device1.getId(), "{\"occupied\":false}"); + Device device3 = createDevice("Device 3", deviceProfile.getId(), "1234567890333"); + + postTelemetry(device3.getId(), "{\"occupied\":true}"); - await().alias("change profile and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + createEntityRelation(asset.getId(), device3.getId(), "Contains"); + + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); assertThat(occupancy).isNotNull(); assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); }); } @Test - public void testAddEntityToProfile_checkMetricsCalculation() throws Exception { - Device device3 = createDevice("Device 3", deviceProfile.getId(), "1234567890333"); - - postTelemetry(device3.getId(), "{\"occupied\":true}"); - - await().alias("add entity to profile and no calculation (there is no relation between device and asset)").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) - .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) - .untilAsserted(this::checkInitialCalculationValues); + public void testDeleteRelation_checkMetricsCalculation() throws Exception { + createOccupancyCF("Occupied spaces", asset.getId()); + checkInitialCalculation(); - createEntityRelation(asset.getId(), device3.getId(), "Contains"); + deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) @@ -197,8 +214,8 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); assertThat(occupancy).isNotNull(); assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); }); } @@ -213,22 +230,22 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset2.getId(), device3.getId(), "Contains"); createEntityRelation(asset2.getId(), device4.getId(), "Contains"); - CalculatedField calculatedField2 = createOccupancyCF("Occupied spaces 2", assetProfile.getId(), List.of(deviceProfile.getId())); + createOccupancyCF("Occupied spaces 2", assetProfile.getId()); await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); - - ObjectNode occupancy2 = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy2).isNotNull(); - assertThat(occupancy2.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancy2.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + ObjectNode occupancyAsset1 = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancyAsset1).isNotNull(); + assertThat(occupancyAsset1.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancyAsset1.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancyAsset1.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + + ObjectNode occupancyAsset2 = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancyAsset2).isNotNull(); + assertThat(occupancyAsset2.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancyAsset2.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancyAsset2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); }); postTelemetry(device3.getId(), "{\"occupied\":true}"); @@ -243,23 +260,26 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll assertThat(occupancy2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); }); } - - - @Test - public void testDeleteRelation_checkMetricsCalculation() throws Exception { - deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); - - await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) - .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) - .untilAsserted(() -> { - ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); - }); - } - +// +// @Test +// public void testChangeProfile_checkMetricsCalculation() throws Exception { +// DeviceProfile deviceProfile2 = doPost("/api/deviceProfile", createDeviceProfile("Device Profile 2"), DeviceProfile.class); +// device1.setDeviceProfileId(deviceProfile2.getId()); +// device1 = doPost("/api/device?accessToken=" + accessToken1, device1, Device.class); +// +// postTelemetry(device1.getId(), "{\"occupied\":false}"); +// +// await().alias("change profile and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) +// .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) +// .untilAsserted(() -> { +// ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); +// assertThat(occupancy).isNotNull(); +// assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); +// assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); +// assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); +// }); +// } +// // @Test // public void testCfWithoutTargetProfileSpecified_checkMetricsCalculation() throws Exception { // Device device3 = createDevice("Device 3", "1234567890333"); @@ -296,18 +316,24 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); } - private CalculatedField createOccupancyCF(String name, EntityId entityId, List profiles) { + private CalculatedField createOccupancyCF(String name, EntityId entityId) { + Map arguments = new HashMap<>(); + Argument argument = new Argument(); + argument.setRefEntityKey(new ReferencedEntityKey("occupied", ArgumentType.TS_LATEST, null)); + argument.setDefaultValue("false"); + arguments.put("oc", argument); + Map aggMetrics = new HashMap<>(); AggMetric freeSpaces = new AggMetric(); freeSpaces.setFunction(AggFunction.COUNT); - freeSpaces.setFilter("return oc == false"); + freeSpaces.setFilter("return oc == false;"); freeSpaces.setInput(new AggKeyInput("oc")); aggMetrics.put("freeSpaces", freeSpaces); AggMetric occupiedSpaces = new AggMetric(); occupiedSpaces.setFunction(AggFunction.COUNT); - occupiedSpaces.setFilter("return oc == true"); + occupiedSpaces.setFilter("return oc == true;"); occupiedSpaces.setInput(new AggKeyInput("oc")); aggMetrics.put("occupiedSpaces", occupiedSpaces); @@ -320,23 +346,16 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll output.setType(OutputType.TIME_SERIES); return createAggCf(name, entityId, - buildSource(EntitySearchDirection.FROM, "Contains", profiles), - Map.of("oc", new ReferencedEntityKey("occupied", ArgumentType.TS_LATEST, null)), + new RelationPathLevel(EntitySearchDirection.FROM, "Contains"), + arguments, aggMetrics, output); } - private AggSource buildSource(EntitySearchDirection direction, String relationType, List profiles) { - AggSource source = new AggSource(); - source.setRelation(new RelationPathLevel(direction, relationType)); - source.setEntityProfiles(profiles); - return source; - } - private CalculatedField createAggCf(String name, EntityId entityId, - AggSource aggSource, - Map inputs, + RelationPathLevel relation, + Map inputs, Map metrics, Output output) { CalculatedField calculatedField = new CalculatedField(); @@ -345,8 +364,8 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll calculatedField.setType(CalculatedFieldType.LATEST_VALUES_AGGREGATION); LatestValuesAggregationCalculatedFieldConfiguration configuration = new LatestValuesAggregationCalculatedFieldConfiguration(); - configuration.setSource(aggSource); - configuration.setInputs(inputs); + configuration.setRelation(relation); + configuration.setArguments(inputs); configuration.setDeduplicationIntervalMillis(deduplicationInterval); configuration.setMetrics(metrics); configuration.setOutput(output); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java deleted file mode 100644 index 65d545bc6e..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.cf.configuration.aggregation; - -import lombok.Builder; -import lombok.Data; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; - -import java.util.List; -import java.util.function.Predicate; - -@Data -@Builder -public class CfAggTrigger { - - private List entityProfiles; - private List inputs; - - public boolean matches(EntityId profileId, Predicate cfAggTrigger) { - if (matchesProfile(profileId)) { - return cfAggTrigger.test(this); - } - return false; - } - - public boolean matchesProfile(EntityId profileId) { - return entityProfiles.isEmpty() || entityProfiles.contains(profileId); - } - - public boolean matchesTimeSeries(List telemetry) { - if (telemetry == null || telemetry.isEmpty()) { - return false; - } - for (TsKvEntry tsKvEntry : telemetry) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKvEntry.getKey(), ArgumentType.TS_LATEST, null); - if (inputs.contains(latestKey)) { - return true; - } - } - return false; - } - - public boolean matchesAttributes(List attributes, AttributeScope scope) { - if (attributes == null || attributes.isEmpty()) { - return false; - } - for (AttributeKvEntry attributeKvEntry : attributes) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(attributeKvEntry.getKey(), ArgumentType.ATTRIBUTE, scope); - if (inputs.contains(latestKey)) { - return true; - } - } - return false; - } - - public boolean matchesTimeSeriesKeys(List telemetry) { - if (telemetry == null || telemetry.isEmpty()) { - return false; - } - - for (String key : telemetry) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); - if (inputs.contains(latestKey)) { - return true; - } - } - - return false; - } - - public boolean matchesAttributesKeys(List attributes, AttributeScope scope) { - if (attributes == null || attributes.isEmpty()) { - return false; - } - - for (String key : attributes) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); - if (inputs.contains(latestKey)) { - return true; - } - } - - return false; - } - -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java index 2760e1855b..89f7e9a339 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -17,18 +17,18 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation; import lombok.Data; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentsBasedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.relation.RelationPathLevel; -import java.util.List; import java.util.Map; @Data -public class LatestValuesAggregationCalculatedFieldConfiguration implements CalculatedFieldConfiguration { +public class LatestValuesAggregationCalculatedFieldConfiguration implements ArgumentsBasedCalculatedFieldConfiguration { - private AggSource source; - private Map inputs; + private RelationPathLevel relation; + private Map arguments; private long deduplicationIntervalMillis; private Map metrics; private Output output; @@ -42,11 +42,4 @@ public class LatestValuesAggregationCalculatedFieldConfiguration implements Calc public void validate() { } - public CfAggTrigger buildTrigger() { - return CfAggTrigger.builder() - .inputs(List.copyOf(inputs.values())) - .entityProfiles(List.copyOf(source.getEntityProfiles())) - .build(); - } - } From 4137a4a615ef4f8cfc84bb82878661feba9db4dd Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 15 Oct 2025 10:05:48 +0300 Subject: [PATCH 04/33] refactoring --- ...CalculatedFieldEntityMessageProcessor.java | 105 ++++++++----- ...alculatedFieldManagerMessageProcessor.java | 139 ++++++------------ .../CalculatedFieldRelatedEntityMsg.java | 9 +- 3 files changed, 117 insertions(+), 136 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index e253633727..63aaf2bd68 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -171,7 +171,8 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM throw cfe; } throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); - } } + } + } public void process(CalculatedFieldArgumentResetMsg msg) throws CalculatedFieldException { log.debug("[{}] Processing CF argument reset msg.", entityId); @@ -202,63 +203,91 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM actorCtx.stop(actorCtx.getSelf()); } } else { - EntityId msgEntityId = msg.getEntityId(); - if (msgEntityId instanceof CalculatedFieldId cfId) { - var state = removeState(cfId); - if (state != null) { - cfStateService.deleteState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); - } else { - msg.getCallback().onSuccess(); - } + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + var state = removeState(cfId); + if (state != null) { + cfStateService.deleteState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); } else { - if (states.isEmpty()) { - msg.getCallback().onSuccess(); - } - for (Map.Entry entry : states.entrySet()) { - LatestValuesAggregationCalculatedFieldState state = (LatestValuesAggregationCalculatedFieldState) entry.getValue(); - state.getArguments().forEach((argName, argEntry) -> { - AggArgumentEntry aggArgEntry = (AggArgumentEntry) argEntry; - aggArgEntry.getAggInputs().remove(msgEntityId); - }); - state.getInputs().remove(msgEntityId); - state.setLastMetricsEvalTs(-1); - processStateIfReady(state, Collections.emptyMap(), state.getCtx(), Collections.emptyList(), null, null, msg.getCallback()); - } + msg.getCallback().onSuccess(); } } } public void process(CalculatedFieldRelatedEntityMsg msg) throws CalculatedFieldException { - log.debug("[{}] Processing CF related entity msg.", msg.getEntityId()); - CalculatedFieldCtx cfCtx = msg.getCalculatedField(); - var state = states.get(cfCtx.getCfId()); - Map fetchedArguments = fetchAggArguments(msg.getCalculatedField(), msg.getEntityId()); + log.debug("[{}] Processing CF {} related entity msg.", msg.getRelatedEntityId(), msg.getAction()); + switch (msg.getAction()) { + case UPDATED -> handleRelationUpdate(msg); + case DELETED -> handleRelationDelete(msg); + default -> msg.getCallback().onSuccess(); + } + } + + private void handleRelationUpdate(CalculatedFieldRelatedEntityMsg msg) throws CalculatedFieldException { + CalculatedFieldCtx ctx = msg.getCalculatedField(); + var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); + var state = states.get(ctx.getCfId()); try { + boolean justRestored = false; if (state == null) { - state = createState(cfCtx); - } else { - state.setCtx(cfCtx, actorCtx); + state = createState(ctx); + justRestored = true; } if (state.isSizeOk()) { - if (state instanceof LatestValuesAggregationCalculatedFieldState latestValuesState) { - latestValuesState.setLastMetricsEvalTs(-1); + Map updatedArgs = new HashMap<>(); + if (!justRestored) { + updatedArgs = updateAggregationState(msg.getRelatedEntityId(), state, ctx); } - state.update(fetchedArguments, cfCtx); - state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, cfCtx.getCfId(), entityId), cfCtx.getMaxStateSize()); - states.put(cfCtx.getCfId(), state); - processStateIfReady(state, fetchedArguments, cfCtx, Collections.singletonList(cfCtx.getCfId()), null, null, msg.getCallback()); + processStateIfReady(state, updatedArgs, ctx, new ArrayList<>(), null, null, callback); } else { - throw new RuntimeException(cfCtx.getSizeExceedsLimitMessage()); + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(); } } catch (Exception e) { - log.debug("[{}][{}] Failed to initialize CF state", entityId, cfCtx.getCfId(), e); + log.debug("[{}][{}] Failed to initialize CF state", entityId, ctx.getCfId(), e); if (e instanceof CalculatedFieldException cfe) { throw cfe; } - throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(entityId).cause(e).build(); + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); + } + } + + private Map updateAggregationState(EntityId relatedEntityId, CalculatedFieldState state, CalculatedFieldCtx ctx) { + Map fetchedArgs = fetchAggArguments(ctx, relatedEntityId); + Map updatedArgs = state.update(fetchedArgs, ctx); + + if (state instanceof LatestValuesAggregationCalculatedFieldState latestValuesState) { + latestValuesState.setLastMetricsEvalTs(-1); + } + + state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); + + return updatedArgs; + } + + private void handleRelationDelete(CalculatedFieldRelatedEntityMsg msg) throws CalculatedFieldException { + CalculatedFieldCtx ctx = msg.getCalculatedField(); + CalculatedFieldId cfId = ctx.getCfId(); + CalculatedFieldState state = states.get(cfId); + if (state == null) { + msg.getCallback().onSuccess(); + return; + } + if (state instanceof LatestValuesAggregationCalculatedFieldState aggState) { + cleanupAggregationState(msg.getRelatedEntityId(), aggState); + processStateIfReady(state, Collections.emptyMap(), state.getCtx(), Collections.emptyList(), null, null, msg.getCallback()); + } else { + msg.getCallback().onSuccess(); } } + private void cleanupAggregationState(EntityId relatedEntityId, LatestValuesAggregationCalculatedFieldState state) { + state.getArguments().values().forEach(argEntry -> { + AggArgumentEntry aggEntry = (AggArgumentEntry) argEntry; + aggEntry.getAggInputs().remove(relatedEntityId); + }); + state.getInputs().remove(relatedEntityId); + state.setLastMetricsEvalTs(-1); + } + @SneakyThrows private Map fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { ListenableFuture> argumentsFuture = cfService.fetchAggEntityArguments(ctx, entityId); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 620eee41ee..37ec4cc3d1 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.actors.calculatedField; import lombok.extern.slf4j.Slf4j; +import org.apache.logging.log4j.util.TriConsumer; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; @@ -29,6 +30,7 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; @@ -303,56 +305,14 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onRelationUpdated(ComponentLifecycleMsg msg, TbCallback callback) { try { - MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); - EntityId toId = entityRelation.getTo(); EntityId fromId = entityRelation.getFrom(); String relationType = entityRelation.getType(); - EntityId toIdProfile = getProfileId(tenantId, toId); - EntityId fromIdProfile = getProfileId(tenantId, fromId); - - List toIdMatches = new ArrayList<>(); - List cfsByToId = getCalculatedFieldsByEntityId(toId); - List cfsByToProfileId = getCalculatedFieldsByEntityId(toIdProfile); - List cfsByToIdOrItsProfileId = new ArrayList<>(); - cfsByToIdOrItsProfileId.addAll(cfsByToId); - cfsByToIdOrItsProfileId.addAll(cfsByToProfileId); - - cfsByToIdOrItsProfileId.forEach(cf -> { - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - RelationPathLevel relation = configuration.getRelation(); - if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType())) { - toIdMatches.add(cf); - } - }); - - MultipleTbCallback toCfsCallback = new MultipleTbCallback(toIdMatches.size(), callbackForToAndFrom); - toIdMatches.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, toCfsCallback, (entityId, cb) -> initRelatedEntity(entityId, fromId, ctx, cb)); - }); - - List fromIdMatches = new ArrayList<>(); - List cfsByFromId = getCalculatedFieldsByEntityId(fromId); - List cfsByFromProfileId = getCalculatedFieldsByEntityId(fromIdProfile); - List cfsByFromIdOrItsProfileId = new ArrayList<>(); - cfsByFromIdOrItsProfileId.addAll(cfsByFromId); - cfsByFromIdOrItsProfileId.addAll(cfsByFromProfileId); - - cfsByFromIdOrItsProfileId.forEach(cf -> { - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - RelationPathLevel relation = configuration.getRelation(); - if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType())) { - fromIdMatches.add(cf); - } - }); - - MultipleTbCallback fromCfsCallback = new MultipleTbCallback(fromIdMatches.size(), callbackForToAndFrom); - fromIdMatches.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, fromCfsCallback, (entityId, cb) -> initRelatedEntity(entityId, toId, ctx, cb)); - }); - + MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); + processRelationByDirection(EntitySearchDirection.TO, relationType, toId, callbackForToAndFrom, (entityId, ctx, cb) -> initRelatedEntity(entityId, fromId, ctx, cb)); + processRelationByDirection(EntitySearchDirection.FROM, relationType, fromId, callbackForToAndFrom, (entityId, ctx, cb) -> initRelatedEntity(entityId, toId, ctx, cb)); } catch (Exception e) { callback.onSuccess(); } @@ -360,59 +320,43 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onRelationDeleted(ComponentLifecycleMsg msg, TbCallback callback) { try { - MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); - EntityId toId = entityRelation.getTo(); EntityId fromId = entityRelation.getFrom(); String relationType = entityRelation.getType(); - EntityId toIdProfile = getProfileId(tenantId, toId); - EntityId fromIdProfile = getProfileId(tenantId, fromId); - - List toIdMatches = new ArrayList<>(); - List cfsByToId = getCalculatedFieldsByEntityId(toId); - List cfsByToProfileId = getCalculatedFieldsByEntityId(toIdProfile); - List cfsByToIdOrItsProfileId = new ArrayList<>(); - cfsByToIdOrItsProfileId.addAll(cfsByToId); - cfsByToIdOrItsProfileId.addAll(cfsByToProfileId); - - cfsByToIdOrItsProfileId.forEach(cf -> { - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - RelationPathLevel relation = configuration.getRelation(); - if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType())) { - toIdMatches.add(cf); - } - }); - MultipleTbCallback toCfsCallback = new MultipleTbCallback(toIdMatches.size(), callbackForToAndFrom); - toIdMatches.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, toCfsCallback, (entityId, cb) -> deleteRelatedEntity(entityId, fromId, cb)); - }); + MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); + processRelationByDirection(EntitySearchDirection.TO, relationType, toId, callbackForToAndFrom, (entityId, ctx, cb) -> deleteRelatedEntity(entityId, fromId, ctx, cb)); + processRelationByDirection(EntitySearchDirection.FROM, relationType, fromId, callbackForToAndFrom, (entityId, ctx, cb) -> deleteRelatedEntity(entityId, toId, ctx, cb)); + } catch (Exception e) { + callback.onSuccess(); + } + } - List fromIdMatches = new ArrayList<>(); - List cfsByFromId = getCalculatedFieldsByEntityId(fromId); - List cfsByFromProfileId = getCalculatedFieldsByEntityId(fromIdProfile); - List cfsByFromIdOrItsProfileId = new ArrayList<>(); - cfsByFromIdOrItsProfileId.addAll(cfsByFromId); - cfsByFromIdOrItsProfileId.addAll(cfsByFromProfileId); - - cfsByFromIdOrItsProfileId.forEach(cf -> { - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - RelationPathLevel relation = configuration.getRelation(); - if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType())) { - fromIdMatches.add(cf); - } - }); + private void processRelationByDirection(EntitySearchDirection direction, + String relationType, + EntityId mainId, + MultipleTbCallback parentCallback, + TriConsumer relationAction) { + List cfsByEntityIdAndProfile = getCalculatedFieldsByEntityIdAndProfile(mainId); + if (cfsByEntityIdAndProfile.isEmpty()) { + parentCallback.onSuccess(); + return; + } - MultipleTbCallback fromCfsCallback = new MultipleTbCallback(fromIdMatches.size(), callbackForToAndFrom); - fromIdMatches.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, fromCfsCallback, (entityId, cb) -> deleteRelatedEntity(entityId, toId, cb)); - }); + List matchingCfs = cfsByEntityIdAndProfile.stream() + .filter(cf -> { + var config = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + RelationPathLevel relation = config.getRelation(); + return direction.equals(relation.direction()) && relationType.equals(relation.relationType()); + }) + .toList(); + MultipleTbCallback directionCallback = new MultipleTbCallback(matchingCfs.size(), parentCallback); - } catch (Exception e) { - callback.onSuccess(); - } + matchingCfs.forEach(ctx -> + applyToTargetCfEntityActors(ctx, directionCallback, (entityId, cb) -> relationAction.accept(entityId, ctx, cb)) + ); } private void onCfCreated(ComponentLifecycleMsg msg, TbCallback callback) throws CalculatedFieldException { @@ -629,9 +573,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware EntityId entityId = msg.getEntityId(); log.debug("Received changed owner msg from entity [{}]", entityId); updateEntityOwner(entityId); - List cfs = new ArrayList<>(); - cfs.addAll(getCalculatedFieldsByEntityId(entityId)); - cfs.addAll(getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId))); + List cfs = getCalculatedFieldsByEntityIdAndProfile(entityId); if (cfs.isEmpty()) { msgCallback.onSuccess(); return; @@ -695,6 +637,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return result; } + private List getCalculatedFieldsByEntityIdAndProfile(EntityId entityId) { + List cfsByEntityIdAndProfile = new ArrayList<>(); + cfsByEntityIdAndProfile.addAll(getCalculatedFieldsByEntityId(entityId)); + cfsByEntityIdAndProfile.addAll(getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId))); + return cfsByEntityIdAndProfile; + } + private List getCalculatedFieldLinksByEntityId(EntityId entityId) { if (entityId == null) { return Collections.emptyList(); @@ -722,14 +671,14 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware getOrCreateActor(entityId).tell(msg); } - private void deleteRelatedEntity(EntityId entityId, EntityId relatedEntityId, TbCallback callback) { + private void deleteRelatedEntity(EntityId entityId, EntityId relatedEntityId, CalculatedFieldCtx cf, TbCallback callback) { log.debug("Pushing delete related entity msg to specific actor [{}]", relatedEntityId); - getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, relatedEntityId, callback)); + getOrCreateActor(entityId).tell(new CalculatedFieldRelatedEntityMsg(tenantId, relatedEntityId, ActionType.DELETED, cf, callback)); } private void initRelatedEntity(EntityId entityId, EntityId relatedEntityId, CalculatedFieldCtx cf, TbCallback callback) { log.debug("Pushing init related entity msg to specific actor [{}]", relatedEntityId); - getOrCreateActor(entityId).tell(new CalculatedFieldRelatedEntityMsg(tenantId, relatedEntityId, cf, callback)); + getOrCreateActor(entityId).tell(new CalculatedFieldRelatedEntityMsg(tenantId, relatedEntityId, ActionType.UPDATED, cf, callback)); } private void deleteCfForEntity(EntityId entityId, CalculatedFieldId cfId, TbCallback callback) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java index d3bdd9cc5c..bf73a1b1b0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java @@ -16,6 +16,7 @@ package org.thingsboard.server.actors.calculatedField; import lombok.Data; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.MsgType; @@ -27,16 +28,18 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; public class CalculatedFieldRelatedEntityMsg implements ToCalculatedFieldSystemMsg { private final TenantId tenantId; - private final EntityId entityId; + private final EntityId relatedEntityId; + private final ActionType action; private final CalculatedFieldCtx calculatedField; private final TbCallback callback; public CalculatedFieldRelatedEntityMsg(TenantId tenantId, - EntityId entityId, + EntityId relatedEntityId, ActionType action, CalculatedFieldCtx calculatedField, TbCallback callback) { this.tenantId = tenantId; - this.entityId = entityId; + this.relatedEntityId = relatedEntityId; + this.action = action; this.calculatedField = calculatedField; this.callback = callback; } From a693cabc05de347138d9116ce01b3c0593543889 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 15 Oct 2025 16:44:36 +0300 Subject: [PATCH 05/33] added more tests --- ...CalculatedFieldEntityMessageProcessor.java | 46 ++- ...alculatedFieldManagerMessageProcessor.java | 6 - ...tractCalculatedFieldProcessingService.java | 3 +- .../service/cf/ctx/state/ArgumentEntry.java | 6 +- .../ctx/state/BaseCalculatedFieldState.java | 11 + .../cf/ctx/state/CalculatedFieldCtx.java | 6 +- .../ctx/state/SimpleCalculatedFieldState.java | 10 - .../ctx/state/SingleValueArgumentEntry.java | 9 + .../state/aggregation/AggArgumentEntry.java | 8 +- ...java => AggSingleEntityArgumentEntry.java} | 17 +- ...ValuesAggregationCalculatedFieldState.java | 103 ++++-- .../aggregation/function/AvgAggEntry.java | 2 +- .../state/aggregation/function/new_agg.json | 4 +- .../utils/CalculatedFieldArgumentUtils.java | 4 +- .../server/utils/CalculatedFieldUtils.java | 10 +- ...tValuesAggregationCalculatedFieldTest.java | 344 ++++++++++++------ ...gregationCalculatedFieldConfiguration.java | 1 + 17 files changed, 388 insertions(+), 202 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/{AggSingleArgumentEntry.java => AggSingleEntityArgumentEntry.java} (79%) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 63aaf2bd68..ce29dc06e0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -53,7 +53,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; @@ -67,6 +67,7 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -329,7 +330,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else if (proto.getAttrDataCount() > 0) { processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); } else if (proto.getRemovedTsKeysCount() > 0) { - processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithFetchedValue(ctx, msg.getEntityId(), proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); } else if (proto.getRemovedAttrKeysCount() > 0) { processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithDefaultValue(ctx, msg.getEntityId(), proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); } else { @@ -405,7 +406,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private void processRemovedTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { - processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithFetchedValue(ctx, entityId, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); } private void processRemovedAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { @@ -565,7 +566,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); String argName = aggArgNames.get(key); if (argName != null) { - arguments.put(argName, new AggSingleArgumentEntry(originator, item)); + arguments.put(argName, new AggSingleEntityArgumentEntry(originator, item)); } } } @@ -618,7 +619,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); String argName = aggArgNames.get(key); if (argName != null) { - arguments.put(argName, new AggSingleArgumentEntry(entityId, item)); + arguments.put(argName, new AggSingleEntityArgumentEntry(entityId, item)); } } } @@ -631,14 +632,21 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return Collections.emptyMap(); } List geofencingArgumentNames = ctx.getLinkedEntityAndCurrentOwnerGeofencingArgumentNames(); - return mapToArgumentsWithDefaultValue(argNames, ctx.getArguments(), geofencingArgumentNames, scope, removedAttrKeys); + List relatedArgumentNames = ctx.getRelatedEntityArgumentNames(); + return mapToArgumentsWithDefaultValue(entityId, argNames, ctx.getArguments(), geofencingArgumentNames, relatedArgumentNames, scope, removedAttrKeys); } private Map mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, AttributeScopeProto scope, List removedAttrKeys) { - return mapToArgumentsWithDefaultValue(ctx.getMainEntityArguments(), ctx.getArguments(), ctx.getMainEntityGeofencingArgumentNames(), scope, removedAttrKeys); + return mapToArgumentsWithDefaultValue(null, ctx.getMainEntityArguments(), ctx.getArguments(), ctx.getMainEntityGeofencingArgumentNames(), new ArrayList<>(), scope, removedAttrKeys); } - private Map mapToArgumentsWithDefaultValue(Map argNames, Map configArguments, List geofencingArgNames, AttributeScopeProto scope, List removedAttrKeys) { + private Map mapToArgumentsWithDefaultValue(EntityId msgEntityId, + Map argNames, + Map configArguments, + List geofencingArgNames, + List relatedEntityArgNames, + AttributeScopeProto scope, + List removedAttrKeys) { Map arguments = new HashMap<>(); for (String removedKey : removedAttrKeys) { ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); @@ -652,22 +660,36 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } Argument argument = configArguments.get(argName); String defaultValue = (argument != null) ? argument.getDefaultValue() : null; - arguments.put(argName, StringUtils.isNotEmpty(defaultValue) + SingleValueArgumentEntry argumentEntry = StringUtils.isNotEmpty(defaultValue) ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) - : new SingleValueArgumentEntry()); + : new SingleValueArgumentEntry(); + if (relatedEntityArgNames.contains(argName)) { + arguments.put(argName, new AggSingleEntityArgumentEntry(msgEntityId, argumentEntry)); + continue; + } + arguments.put(argName, argumentEntry); } return arguments; } - private Map mapToArgumentsWithFetchedValue(CalculatedFieldCtx ctx, List removedTelemetryKeys) { + private Map mapToArgumentsWithFetchedValue(CalculatedFieldCtx ctx, EntityId entityId, List removedTelemetryKeys) { Map deletedArguments = ctx.getArguments().entrySet().stream() .filter(entry -> removedTelemetryKeys.contains(entry.getValue().getRefEntityKey().getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); - fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); + if (CalculatedFieldType.LATEST_VALUES_AGGREGATION.equals(ctx.getCfType())) { + fetchedArgs = fetchedArgs.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + argEntry -> new AggSingleEntityArgumentEntry(entityId, argEntry.getValue()) + )); + } else { + fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); + } + return fetchedArgs; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 37ec4cc3d1..af5a672157 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -516,12 +516,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware .toList(); } - private List getCfsWithRelationToEntity(EntityId entityId) { - return aggCalculatedFields.values().stream() - .filter(cf -> !findRelationsForCf(entityId, cf).isEmpty()) - .toList(); - } - private List findRelationsForCf(EntityId entityId, CalculatedFieldCtx cf) { List result = new ArrayList<>(); if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration configuration) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index c30a12a8f3..94695bcc8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -250,7 +250,8 @@ public abstract class AbstractCalculatedFieldProcessingService { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))), MoreExecutors.directExecutor()); } - public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) {List>> futures = aggEntities.stream() + public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) { + List>> futures = aggEntities.stream() .map(entityId -> { ListenableFuture singleAggEntryFut = fetchSingleAggArgumentEntry(tenantId, entityId, argument, startTs); return Futures.transform(singleAggEntryFut, singleAggEntry -> Map.entry(entityId, singleAggEntry), MoreExecutors.directExecutor()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 4e3d00ee62..ca0a12e1d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import java.util.List; @@ -39,7 +39,7 @@ import java.util.Map; @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING"), @JsonSubTypes.Type(value = AggArgumentEntry.class, name = "AGGREGATE_LATEST"), - @JsonSubTypes.Type(value = AggSingleArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") + @JsonSubTypes.Type(value = AggSingleEntityArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") }) public interface ArgumentEntry { @@ -75,7 +75,7 @@ public interface ArgumentEntry { } static ArgumentEntry createAggSingleArgument(EntityId entityId, KvEntry kvEntry) { - return new AggSingleArgumentEntry(entityId, kvEntry); + return new AggSingleEntityArgumentEntry(entityId, kvEntry); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index f75711a107..2d853fd3fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Getter; import lombok.Setter; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -134,4 +135,14 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, this.latestTimestamp = Math.max(this.latestTimestamp, newTs); } + protected Object formatResult(double result, Integer decimals) { + if (decimals == null) { + return result; + } + if (decimals.equals(0)) { + return TbUtils.toInt(result); + } + return TbUtils.toFixed(result, decimals); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index f8b9f5b665..3a9d75b3d4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -107,6 +107,7 @@ public class CalculatedFieldCtx { private boolean relationQueryDynamicArguments; private List mainEntityGeofencingArgumentNames; private List linkedEntityAndCurrentOwnerGeofencingArgumentNames; + private List relatedEntityArgumentNames; private long scheduledUpdateIntervalMillis; @@ -126,6 +127,7 @@ public class CalculatedFieldCtx { this.argNames = new ArrayList<>(); this.mainEntityGeofencingArgumentNames = new ArrayList<>(); this.linkedEntityAndCurrentOwnerGeofencingArgumentNames = new ArrayList<>(); + this.relatedEntityArgumentNames = new ArrayList<>(); this.output = calculatedField.getConfiguration().getOutput(); if (calculatedField.getConfiguration() instanceof ArgumentsBasedCalculatedFieldConfiguration argBasedConfig) { this.arguments.putAll(argBasedConfig.getArguments()); @@ -153,6 +155,7 @@ public class CalculatedFieldCtx { } } this.argNames.addAll(arguments.keySet()); + this.relatedEntityArgumentNames.addAll(relatedEntityArguments.values()); if (argBasedConfig instanceof ExpressionBasedCalculatedFieldConfiguration expressionBasedConfig) { this.expression = expressionBasedConfig.getExpression(); this.useLatestTs = CalculatedFieldType.SIMPLE.equals(calculatedField.getType()) && ((SimpleCalculatedFieldConfiguration) argBasedConfig).isUseLatestTs(); @@ -174,7 +177,7 @@ public class CalculatedFieldCtx { } this.requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - this.scheduledUpdateIntervalMillis = aggConfig.getDeduplicationIntervalMillis(); + this.useLatestTs = aggConfig.isUseLatestTs(); } this.systemContext = systemContext; this.tbelInvokeService = systemContext.getTbelInvokeService(); @@ -578,6 +581,7 @@ public class CalculatedFieldCtx { } if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig && other.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig + && thisConfig.getDeduplicationIntervalMillis() != otherConfig.getDeduplicationIntervalMillis() && !thisConfig.getMetrics().equals(otherConfig.getMetrics())) { return true; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 65cb595632..8886482ef8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -62,16 +62,6 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { .build()); } - private Object formatResult(double expressionResult, Integer decimals) { - if (decimals == null) { - return expressionResult; - } - if (decimals.equals(0)) { - return TbUtils.toInt(expressionResult); - } - return TbUtils.toFixed(expressionResult, decimals); - } - private JsonNode createResultJson(boolean useLatestTs, String outputName, Object result) { ObjectNode valuesNode = JacksonUtil.newObjectNode(); if (result instanceof Double doubleValue) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 288b486e83..e81201c961 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -45,6 +45,15 @@ public class SingleValueArgumentEntry implements ArgumentEntry { public static final Long DEFAULT_VERSION = -1L; + public SingleValueArgumentEntry(ArgumentEntry entry) { + if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + this.ts = singleValueArgumentEntry.ts; + this.kvEntryValue = singleValueArgumentEntry.kvEntryValue; + this.version = singleValueArgumentEntry.version; + this.forceResetPrevious = singleValueArgumentEntry.forceResetPrevious; + } + } + public SingleValueArgumentEntry(TsKvProto entry) { this.ts = entry.getTs(); if (entry.hasVersion()) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java index 12ae2c4638..7e3a8623e4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java @@ -48,11 +48,11 @@ public class AggArgumentEntry implements ArgumentEntry { if (entry instanceof AggArgumentEntry aggArgumentEntry) { aggInputs.putAll(aggArgumentEntry.aggInputs); return true; - } else if (entry instanceof AggSingleArgumentEntry aggSingleArgumentEntry) { - if (aggSingleArgumentEntry.isDeleted()) { - aggInputs.remove(aggSingleArgumentEntry.getEntityId()); + } else if (entry instanceof AggSingleEntityArgumentEntry aggSingleEntityArgumentEntry) { + if (aggSingleEntityArgumentEntry.isDeleted()) { + aggInputs.remove(aggSingleEntityArgumentEntry.getEntityId()); } else { - aggInputs.put(aggSingleArgumentEntry.getEntityId(), aggSingleArgumentEntry); + aggInputs.put(aggSingleEntityArgumentEntry.getEntityId(), aggSingleEntityArgumentEntry); } return true; } else { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java index 6b81d5380c..32ce77311d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java @@ -30,34 +30,39 @@ import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; @Data @NoArgsConstructor @AllArgsConstructor -public class AggSingleArgumentEntry extends SingleValueArgumentEntry { +public class AggSingleEntityArgumentEntry extends SingleValueArgumentEntry { private EntityId entityId; private boolean deleted; - public AggSingleArgumentEntry(EntityId entityId, TsKvProto entry) { + public AggSingleEntityArgumentEntry(EntityId entityId, ArgumentEntry entry) { super(entry); this.entityId = entityId; } - public AggSingleArgumentEntry(EntityId entityId, AttributeValueProto entry) { + public AggSingleEntityArgumentEntry(EntityId entityId, TsKvProto entry) { super(entry); this.entityId = entityId; } - public AggSingleArgumentEntry(EntityId entityId, KvEntry entry) { + public AggSingleEntityArgumentEntry(EntityId entityId, AttributeValueProto entry) { super(entry); this.entityId = entityId; } - public AggSingleArgumentEntry(EntityId entityId, long ts, BasicKvEntry kvEntryValue, Long version) { + public AggSingleEntityArgumentEntry(EntityId entityId, KvEntry entry) { + super(entry); + this.entityId = entityId; + } + + public AggSingleEntityArgumentEntry(EntityId entityId, long ts, BasicKvEntry kvEntryValue, Long version) { super(ts, kvEntryValue, version); this.entityId = entityId; } @Override public boolean updateEntry(ArgumentEntry entry) { - if (entry instanceof AggSingleArgumentEntry singleValueEntry) { + if (entry instanceof AggSingleEntityArgumentEntry singleValueEntry) { if (singleValueEntry.getTs() <= ts) { return false; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java index d25cde020c..1df27b4cbd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import lombok.Data; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.TbActorRef; @@ -43,10 +45,11 @@ import java.util.Map; import java.util.Map.Entry; @Slf4j -@Data +@Getter public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedFieldState { private long lastArgsRefreshTs = -1; + @Setter private long lastMetricsEvalTs = -1; private long deduplicationInterval = -1; private Map metrics; @@ -76,8 +79,7 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF @Override public void init() { super.init(); -// long scheduledUpdateIntervalMillis = ctx.getScheduledUpdateIntervalMillis(); -// ctx.scheduleReevaluation(scheduledUpdateIntervalMillis, actorCtx); + ctx.scheduleReevaluation(deduplicationInterval, actorCtx); } @Override @@ -102,41 +104,51 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception { + if (!shouldRecalculate()) { + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .result(null) + .build()); + } + Output output = ctx.getOutput(); + ObjectNode aggResult = aggregateMetrics(output); + lastMetricsEvalTs = System.currentTimeMillis(); + ctx.scheduleReevaluation(deduplicationInterval, actorCtx); + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .type(output.getType()) + .scope(output.getScope()) + .result(createResultJson(ctx.isUseLatestTs(), aggResult)) + .build()); + } + + private boolean shouldRecalculate() { boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - deduplicationInterval; boolean argsUpdatedDuringInterval = lastArgsRefreshTs > lastMetricsEvalTs; - if (intervalPassed && argsUpdatedDuringInterval) { - ObjectNode aggResult = JacksonUtil.newObjectNode(); - for (Entry entry : metrics.entrySet()) { - String metricKey = entry.getKey(); - AggMetric metric = entry.getValue(); - - AggEntry aggMetric = AggFunctionFactory.createAggFunction(metric.getFunction()); - - for (Map entityInputs : inputs.values()) { - if (applyAggregation(metric.getFilter(), entityInputs)) { - Object arg = resolveAggregationInput(metric.getInput(), entityInputs); - if (arg != null) { - aggMetric.update(arg); - } - } - } + return intervalPassed && argsUpdatedDuringInterval; + } + + private ObjectNode aggregateMetrics(Output output) throws Exception { + ObjectNode aggResult = JacksonUtil.newObjectNode(); + for (Entry entry : metrics.entrySet()) { + String metricKey = entry.getKey(); + AggMetric metric = entry.getValue(); - aggMetric.result().ifPresent(result -> { - aggResult.set(metricKey, JacksonUtil.valueToTree(result)); - }); + AggEntry aggMetricEntry = AggFunctionFactory.createAggFunction(metric.getFunction()); + aggregateMetric(metric, aggMetricEntry); + aggMetricEntry.result().ifPresent(result -> { + aggResult.set(metricKey, JacksonUtil.valueToTree(formatResult(result, output.getDecimalsByDefault()))); + }); + } + return aggResult; + } + + private void aggregateMetric(AggMetric metric, AggEntry aggEntry) throws Exception { + for (Map entityInputs : inputs.values()) { + if (applyAggregation(metric.getFilter(), entityInputs)) { + Object arg = resolveAggregationInput(metric.getInput(), entityInputs); + if (arg != null) { + aggEntry.update(arg); + } } - Output output = ctx.getOutput(); - lastMetricsEvalTs = System.currentTimeMillis(); - ctx.scheduleReevaluation(deduplicationInterval, actorCtx); - return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() - .type(output.getType()) - .scope(output.getScope()) - .result(aggResult) - .build()); - } else { - return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() - .result(null) - .build()); } } @@ -158,4 +170,25 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF } } + private Object formatResult(Object aggregationResult, Integer decimals) { + try { + double result = Double.parseDouble(aggregationResult.toString()); + return formatResult(result, decimals); + } catch (Exception e) { + throw new IllegalArgumentException("Aggregation result cannot be parsed: " + aggregationResult, e); + } + } + + protected JsonNode createResultJson(boolean useLatestTs, JsonNode result) { + long latestTs = getLatestTimestamp(); + if (useLatestTs && latestTs != -1) { + ObjectNode resultNode = JacksonUtil.newObjectNode(); + resultNode.put("ts", latestTs); + resultNode.set("values", result); + return resultNode; + } else { + return result; + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java index ad1f2ee8a8..afe6abb93e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java @@ -35,7 +35,7 @@ public class AvgAggEntry extends BaseAggEntry { @Override protected double prepareResult() { - return sum.divide(BigDecimal.valueOf(count), 2, RoundingMode.HALF_UP).doubleValue(); + return sum.divide(BigDecimal.valueOf(count), 10, RoundingMode.HALF_UP).doubleValue(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json index 32cd053d41..c6b841b673 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json @@ -7,8 +7,8 @@ "allEnabledUntil": 1769907492297 }, "entityId": { - "entityType": "ASSET_PROFILE", - "id": "2b759c60-a8f4-11f0-be29-7fa922118588" + "entityType": "ASSET", + "id": "f8ad0800-a9a6-11f0-bbe6-459b63b420fe" }, "configuration": { "type": "LATEST_VALUES_AGGREGATION", diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index 6485508602..74cdb0cddb 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -34,7 +34,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -59,7 +59,7 @@ public class CalculatedFieldArgumentUtils { if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { return ArgumentEntry.createAggSingleArgument(entityId, kvEntry.get()); } else { - return new AggSingleArgumentEntry(); + return new AggSingleEntityArgumentEntry(); } } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index d4ee7bb2e3..19ad4bfb7c 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -47,7 +47,7 @@ import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; @@ -236,7 +236,7 @@ public class CalculatedFieldUtils { LatestValuesAggregationCalculatedFieldState aggState = (LatestValuesAggregationCalculatedFieldState) state; Map> arguments = new HashMap<>(); proto.getAggArgumentsList().forEach(argProto -> { - AggSingleArgumentEntry entry = fromAggSingleValueArgumentProto(argProto); + AggSingleEntityArgumentEntry entry = fromAggSingleValueArgumentProto(argProto); arguments.computeIfAbsent(argProto.getValue().getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); }); arguments.forEach((argName, entityInputs) -> { @@ -248,14 +248,14 @@ public class CalculatedFieldUtils { return state; } - public static AggSingleArgumentEntry fromAggSingleValueArgumentProto(AggSingleArgumentEntryProto proto) { + public static AggSingleEntityArgumentEntry fromAggSingleValueArgumentProto(AggSingleArgumentEntryProto proto) { if (!proto.hasValue()) { - return new AggSingleArgumentEntry(); + return new AggSingleEntityArgumentEntry(); } EntityId entityId = ProtoUtils.fromProto(proto.getEntityId()); SingleValueArgumentProto singleValueArgument = proto.getValue(); TsValueProto tsValueProto = singleValueArgument.getValue(); - return new AggSingleArgumentEntry( + return new AggSingleEntityArgumentEntry( entityId, tsValueProto.getTs(), (BasicKvEntry) KvProtoUtil.fromTsValueProto(singleValueArgument.getArgName(), tsValueProto), diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java index cd9c1c578b..a4b43e362b 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -58,6 +58,7 @@ import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.cf.CalculatedFieldIntegrationTest.POLL_INTERVAL; @DaoSqlTest @@ -116,7 +117,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll } @Test - public void testNoTelemetryOnDevices_checkDefaultValueUsed() throws Exception { + public void testCreateCfOnProfile_checkInitialAggregation() throws Exception { Asset asset2 = createAsset("Asset 2", assetProfile.getId()); Device device3 = createDevice("Device 3", "1234567890333"); Device device4 = createDevice("Device 4", "1234567890444"); @@ -124,22 +125,153 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset2.getId(), device3.getId(), "Contains"); createEntityRelation(asset2.getId(), device4.getId(), "Contains"); - createOccupancyCF("Occupied spaces", asset2.getId()); + createOccupancyCF(assetProfile.getId()); - await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "1", + "totalSpaces", "2" + )); + + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); + }); + + postTelemetry(device3.getId(), "{\"occupied\":true}"); + + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "1", + "totalSpaces", "2" + )); + }); + } + + @Test + public void testAddEntityToProfile_checkAggregation() throws Exception { + createOccupancyCF(assetProfile.getId()); + + Device device3 = createDevice("Device 3", "1234567890333"); + Device device4 = createDevice("Device 4", "1234567890444"); + postTelemetry(device3.getId(), "{\"occupied\":true}"); + postTelemetry(device4.getId(), "{\"occupied\":true}"); + + Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + + await().alias("add entity to profile with no related entities and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode occupancy = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("freeSpaces").get(0).get("value").isNull()).isTrue(); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").isNull()).isTrue(); + assertThat(occupancy.get("totalSpaces").get(0).get("value").isNull()).isTrue(); + }); + + createEntityRelation(asset2.getId(), device3.getId(), "Contains"); + createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + + await().alias("create relations and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "0", + "occupiedSpaces", "2", + "totalSpaces", "2" + )); + }); + + postTelemetry(device3.getId(), "{\"occupied\":false}"); + + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "1", + "totalSpaces", "2" + )); }); } @Test - public void testUpdateTelemetry_checkMetricsCalculation() throws Exception { - createOccupancyCF("Occupied spaces", asset.getId()); + public void testChangeEntityProfile_checkAggregation() throws Exception { + Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + Device device3 = createDevice("Device 3", "1234567890333"); + Device device4 = createDevice("Device 4", "1234567890444"); + + createEntityRelation(asset2.getId(), device3.getId(), "Contains"); + createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + + createOccupancyCF(assetProfile.getId()); + + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "1", + "totalSpaces", "2" + )); + + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); + }); + + AssetProfile newAssetProfile = createAssetProfile("New Asset Profile"); + asset2.setAssetProfileId(newAssetProfile.getId()); + doPost("/api/asset", asset2, Asset.class); + + postTelemetry(device3.getId(), "{\"occupied\":true}"); + + await().alias("change profile and no aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); + }); + } + + @Test + public void testCreateCfOnAssetAndNoTelemetryOnDevices_checkDefaultValueUsed() throws Exception { + Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + Device device3 = createDevice("Device 3", "1234567890333"); + Device device4 = createDevice("Device 4", "1234567890444"); + + createEntityRelation(asset2.getId(), device3.getId(), "Contains"); + createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + + createOccupancyCF(asset2.getId()); + + await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); + }); + } + + @Test + public void testCreateCfAndUpdateTelemetry_checkAggregation() throws Exception { + createOccupancyCF(asset.getId()); checkInitialCalculation(); postTelemetry(device1.getId(), "{\"occupied\":false}"); @@ -147,17 +279,38 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); }); } @Test - public void testUpdateTelemetry_checkMetricsCalculationNotExecutedUntilDeduplicationInterval() throws Exception { - createOccupancyCF("Occupied spaces", asset.getId()); + public void testDeleteCf_checkNoAggregation() throws Exception { + CalculatedField cf = createOccupancyCF(asset.getId()); + checkInitialCalculation(); + + doDelete("/api/calculatedField/" + cf.getId().getId().toString()) + .andExpect(status().isOk()); + + postTelemetry(device1.getId(), "{\"occupied\":false}"); + + await().alias("delete cf and update telemetry and no aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "1", + "totalSpaces", "2" + )); + }); + } + + @Test + public void testUpdateTelemetry_checkAggregationNotExecutedUntilDeduplicationInterval() throws Exception { + createOccupancyCF(asset.getId()); checkInitialCalculation(); postTelemetry(device1.getId(), "{\"occupied\":false}"); @@ -171,136 +324,92 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll await().alias("create CF and perform initial calculation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); }); } @Test - public void testCreateRelation_checkMetricsCalculation() throws Exception { - createOccupancyCF("Occupied spaces", asset.getId()); - checkInitialCalculation(); + public void testDeleteTelemetry_checkAggregationWithPreviousValuesOrDefault() throws Exception { + Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + Device device3 = createDevice("Device 3", "1234567890333"); + Device device4 = createDevice("Device 4", "1234567890444"); - Device device3 = createDevice("Device 3", deviceProfile.getId(), "1234567890333"); + createEntityRelation(asset2.getId(), device3.getId(), "Contains"); + createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + postTelemetry(device3.getId(), "{\"occupied\":false}"); + postTelemetry(device4.getId(), "{\"occupied\":true}"); postTelemetry(device3.getId(), "{\"occupied\":true}"); - createEntityRelation(asset.getId(), device3.getId(), "Contains"); + createOccupancyCF(asset2.getId()); - await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "0", + "occupiedSpaces", "2", + "totalSpaces", "2" + )); }); - } - - @Test - public void testDeleteRelation_checkMetricsCalculation() throws Exception { - createOccupancyCF("Occupied spaces", asset.getId()); - checkInitialCalculation(); - deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); + doDelete("/api/plugins/telemetry/DEVICE/" + device3.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=0&endTs=" + System.currentTimeMillis(), String.class); + doDelete("/api/plugins/telemetry/DEVICE/" + device4.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=0&endTs=" + System.currentTimeMillis(), String.class); - await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("delete latest telemetry and perform aggregation with previous or default values").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); }); } @Test - public void testCfOnProfile_checkMetricsCalculation() throws Exception { - Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + public void testCreateRelation_checkAggregation() throws Exception { + createOccupancyCF(asset.getId()); + checkInitialCalculation(); + Device device3 = createDevice("Device 3", deviceProfile.getId(), "1234567890333"); - postTelemetry(device3.getId(), "{\"occupied\":false}"); - Device device4 = createDevice("Device 4", deviceProfile.getId(), "1234567890444"); - postTelemetry(device4.getId(), "{\"occupied\":false}"); - createEntityRelation(asset2.getId(), device3.getId(), "Contains"); - createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + postTelemetry(device3.getId(), "{\"occupied\":true}"); - createOccupancyCF("Occupied spaces 2", assetProfile.getId()); + createEntityRelation(asset.getId(), device3.getId(), "Contains"); - await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancyAsset1 = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancyAsset1).isNotNull(); - assertThat(occupancyAsset1.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancyAsset1.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancyAsset1.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); - - ObjectNode occupancyAsset2 = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancyAsset2).isNotNull(); - assertThat(occupancyAsset2.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancyAsset2.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancyAsset2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "2", + "totalSpaces", "3" + )); }); + } - postTelemetry(device3.getId(), "{\"occupied\":true}"); + @Test + public void testDeleteRelation_checkAggregation() throws Exception { + createOccupancyCF(asset.getId()); + checkInitialCalculation(); - await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) + deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); + + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancy2 = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy2).isNotNull(); - assertThat(occupancy2.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy2.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "0", + "totalSpaces", "1" + )); }); } -// -// @Test -// public void testChangeProfile_checkMetricsCalculation() throws Exception { -// DeviceProfile deviceProfile2 = doPost("/api/deviceProfile", createDeviceProfile("Device Profile 2"), DeviceProfile.class); -// device1.setDeviceProfileId(deviceProfile2.getId()); -// device1 = doPost("/api/device?accessToken=" + accessToken1, device1, Device.class); -// -// postTelemetry(device1.getId(), "{\"occupied\":false}"); -// -// await().alias("change profile and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) -// .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) -// .untilAsserted(() -> { -// ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); -// assertThat(occupancy).isNotNull(); -// assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); -// assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); -// assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); -// }); -// } -// -// @Test -// public void testCfWithoutTargetProfileSpecified_checkMetricsCalculation() throws Exception { -// Device device3 = createDevice("Device 3", "1234567890333"); -// postTelemetry(device3.getId(), "{\"occupied\":true}"); -// createEntityRelation(asset.getId(), device3.getId(), "Contains"); -// -// var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) calculatedField.getConfiguration(); -// configuration.getSource().setEntityProfiles(Collections.emptyList()); -// calculatedField.setConfiguration(configuration); -// saveCalculatedField(calculatedField); -// -// await().alias("update cf and perform aggregation for 3 devices").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) -// .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) -// .untilAsserted(() -> { -// ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); -// assertThat(occupancy).isNotNull(); -// assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); -// assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); -// assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); -// }); -// } private void checkInitialCalculation() { await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) @@ -316,7 +425,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); } - private CalculatedField createOccupancyCF(String name, EntityId entityId) { + private CalculatedField createOccupancyCF(EntityId entityId) { Map arguments = new HashMap<>(); Argument argument = new Argument(); argument.setRefEntityKey(new ReferencedEntityKey("occupied", ArgumentType.TS_LATEST, null)); @@ -344,8 +453,9 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll Output output = new Output(); output.setType(OutputType.TIME_SERIES); + output.setDecimalsByDefault(0); - return createAggCf(name, entityId, + return createAggCf("Occupied spaces", entityId, new RelationPathLevel(EntitySearchDirection.FROM, "Contains"), arguments, aggMetrics, @@ -393,6 +503,12 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll return doPost("/api/asset", asset, Asset.class); } + private void verifyTelemetry(EntityId entityId, Map expectedResults) throws Exception { + ObjectNode result = getLatestTelemetry(entityId, expectedResults.keySet().toArray(new String[0])); + assertThat(result).isNotNull(); + expectedResults.forEach((key, value) -> assertThat(result.get(key).get(0).get("value").asText()).isEqualTo(value)); + } + private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java index 89f7e9a339..43a3360ce6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -32,6 +32,7 @@ public class LatestValuesAggregationCalculatedFieldConfiguration implements Argu private long deduplicationIntervalMillis; private Map metrics; private Output output; + private boolean useLatestTs; @Override public CalculatedFieldType getType() { From e6479d5856e4c8860e674d2d309a502c978aa25b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 16 Oct 2025 12:38:37 +0300 Subject: [PATCH 06/33] removed relation by profile processing --- .../processing/AbstractConsumerService.java | 16 +-- .../server/dao/relation/RelationService.java | 15 --- .../configuration/aggregation/AggSource.java | 30 ----- .../ProfileEntityRelationPathQuery.java | 21 ---- .../dao/relation/BaseRelationService.java | 100 ---------------- .../server/dao/relation/RelationCacheKey.java | 7 +- .../server/dao/relation/RelationDao.java | 7 -- .../dao/sql/relation/JpaRelationDao.java | 108 ------------------ .../dao/sql/relation/RelationRepository.java | 36 ------ 9 files changed, 5 insertions(+), 335 deletions(-) delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 36a35d9e44..6e162256a4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -188,13 +188,9 @@ public abstract class AbstractConsumerService> findByRelationPathQueryAsync(TenantId tenantId, EntityRelationPathQuery relationPathQuery); - ListenableFuture> findByProfileEntityRelationPathQueryAsync(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery); - - List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery); - - ListenableFuture> findByFromAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId from, String relationType, EntityId targetProfileId); - - List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId profileId); - - ListenableFuture> findByToAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId to, String relationType, EntityId targetProfileId); - - List findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId); - - void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId); - // TODO: This method may be useful for some validations in the future // ListenableFuture checkRecursiveRelation(EntityId from, EntityId to); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java deleted file mode 100644 index 84f6f92eb3..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.cf.configuration.aggregation; - -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.relation.RelationPathLevel; - -import java.util.List; - -@Data -public class AggSource { - - private RelationPathLevel relation; - private List entityProfiles; - -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java deleted file mode 100644 index 32b338ff6f..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.relation; - -import org.thingsboard.server.common.data.id.EntityId; - -public record ProfileEntityRelationPathQuery(EntityId rootEntityId, RelationPathLevel level, EntityId targetEntityProfileId) { -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 2d584ebebe..18d1806fe4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -45,7 +45,6 @@ import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; -import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.relation.RelationTypeGroup; @@ -515,105 +514,6 @@ public class BaseRelationService implements RelationService { return executor.submit(() -> relationDao.findByRelationPathQuery(tenantId, relationPathQuery)); } - @Override - public ListenableFuture> findByProfileEntityRelationPathQueryAsync(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery) { - log.trace("Executing findByProfileEntityRelationPathQueryAsync, tenantId [{}], relationPathQuery {}", tenantId, relationPathQuery); - validateId(tenantId, id -> "Invalid tenant id: " + id); - validate(relationPathQuery); - RelationPathLevel relationPathLevel = relationPathQuery.level(); - return switch (relationPathLevel.direction()) { - case FROM -> findByFromAndTypeAndEntityProfileAsync(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); - case TO -> findByToAndTypeAndEntityProfileAsync(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); - }; - } - - @Override - public List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery) { - log.trace("Executing findByProfileEntityRelationPathQuery, tenantId [{}], relationPathQuery {}", tenantId, relationPathQuery); - validateId(tenantId, id -> "Invalid tenant id: " + id); - validate(relationPathQuery); - return relationDao.findByProfileEntityRelationPathQuery(tenantId, relationPathQuery); -// RelationPathLevel relationPathLevel = relationPathQuery.level(); -// return switch (relationPathLevel.direction()) { -// case FROM -> findByFromAndTypeAndEntityProfile(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); -// case TO -> findByToAndTypeAndEntityProfile(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); -// }; - } - - @Override - public ListenableFuture> findByFromAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId from, String relationType, EntityId targetProfileId) { - log.trace("Executing findByFromAndTypeAndEntityProfileAsync [{}][{}][{}]", from, relationType, targetProfileId); - validate(from); - validateType(relationType); - if (targetProfileId == null) { - return findByFromAndTypeAsync(tenantId, from, relationType, RelationTypeGroup.COMMON); - } - return executor.submit(() -> findByFromAndTypeAndEntityProfile(tenantId, from, relationType, targetProfileId)); - } - - @Override - public List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId targetProfileId) { - if (targetProfileId == null) { - return findByFromAndType(tenantId, from, relationType, RelationTypeGroup.COMMON); - } -// RelationCacheKey cacheKey = RelationCacheKey.builder().from(from).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.FROM).entityProfile(targetProfileId).build(); -// return cache.getAndPutInTransaction(cacheKey, -// () -> relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, targetProfileId), -// RelationCacheValue::getRelations, -// relations -> RelationCacheValue.builder().relations(relations).build(), false); - - return relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, targetProfileId); - } - - @Override - public ListenableFuture> findByToAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId to, String relationType, EntityId targetProfileId) { - log.trace("Executing findByToAndTypeAndEntityProfileAsync [{}][{}][{}]", to, relationType, targetProfileId); - validate(to); - validateType(relationType); - if (targetProfileId == null) { - return findByToAndTypeAsync(tenantId, to, relationType, RelationTypeGroup.COMMON); - } - return executor.submit(() -> findByToAndTypeAndEntityProfile(tenantId, to, relationType, targetProfileId)); - } - - @Override - public List findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId targetProfileId) { - if (targetProfileId == null) { - return findByFromAndType(tenantId, to, relationType, RelationTypeGroup.COMMON); - } -// RelationCacheKey cacheKey = RelationCacheKey.builder().to(to).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.TO).entityProfile(targetProfileId).build(); -// return cache.getAndPutInTransaction(cacheKey, -// () -> relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, targetProfileId), -// RelationCacheValue::getRelations, -// relations -> RelationCacheValue.builder().relations(relations).build(), false); - - return relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, targetProfileId); - } - - @Override - public void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId) { - -// List keys = new ArrayList<>(5); -// keys.add(new RelationCacheKey(entityId, null, event.getType(), event.getTypeGroup())); -// keys.add(new RelationCacheKey(event.getFrom(), null, event.getType(), event.getTypeGroup(), EntitySearchDirection.FROM)); -// keys.add(new RelationCacheKey(event.getFrom(), null, null, event.getTypeGroup(), EntitySearchDirection.FROM)); -// keys.add(new RelationCacheKey(null, event.getTo(), event.getType(), event.getTypeGroup(), EntitySearchDirection.TO)); -// keys.add(new RelationCacheKey(null, event.getTo(), null, event.getTypeGroup(), EntitySearchDirection.TO)); -// cache.evict(keys); -// log.debug("Processed evict event: {}", event); - - List keys = new ArrayList<>(2); - keys.add(RelationCacheKey.builder().from(entityId).entityProfile(profileId).build()); - keys.add(RelationCacheKey.builder().to(entityId).entityProfile(profileId).build()); - cache.evict(keys); - log.debug("Processed evict relations by keys: {}", keys); - } - - private void validate(ProfileEntityRelationPathQuery relationPathQuery) { - validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); - relationPathQuery.level().validate(); - } - private void validate(EntityRelationPathQuery relationPathQuery) { validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); List levels = relationPathQuery.levels(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java index 344af0a6f3..d6f0525c9d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java @@ -40,14 +40,9 @@ public class RelationCacheKey implements Serializable { private final String type; private final RelationTypeGroup typeGroup; private final EntitySearchDirection direction; - private final EntityId entityProfile; public RelationCacheKey(EntityId from, EntityId to, String type, RelationTypeGroup typeGroup) { - this(from, to, type, typeGroup, null, null); - } - - public RelationCacheKey(EntityId from, EntityId to, String type, RelationTypeGroup typeGroup, EntitySearchDirection direction) { - this(from, to, type, typeGroup, direction, null); + this(from, to, type, typeGroup, null); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index 6318f1fc9d..ad53164ad7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -20,7 +20,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; -import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -37,12 +36,8 @@ public interface RelationDao { List findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup); - List findByFromAndTypeAndProfile(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup, EntityId profileId); - List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup); - List findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId); - List findAllByTo(TenantId tenantId, EntityId to); List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup); @@ -79,6 +74,4 @@ public interface RelationDao { List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery relationPathQuery); - List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery query); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index afecca26c1..b2871313ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; -import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -42,12 +41,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; -import static org.thingsboard.server.dao.model.ModelConstants.ASSET_TABLE_NAME; -import static org.thingsboard.server.dao.model.ModelConstants.DEVICE_TABLE_NAME; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_FROM_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_FROM_TYPE_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_TABLE_NAME; @@ -107,11 +103,6 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } - @Override - public List findByFromAndTypeAndProfile(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { - return DaoUtil.convertDataList(relationRepository.findByFromAndProfile(from.getId(), from.getEntityType().name(), typeGroup.name(), relationType, profileId.getId())); - } - @Override public List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup) { return DaoUtil.convertDataList( @@ -121,17 +112,6 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } - @Override - public List findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { - return DaoUtil.convertDataList( - relationRepository.findByToAndProfile( - to.getId(), - to.getEntityType().name(), - typeGroup.name(), - relationType, - profileId.getId())); - } - @Override public List findAllByTo(TenantId tenantId, EntityId to) { return DaoUtil.convertDataList( @@ -412,92 +392,4 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple return sb.toString(); } - @Override - public List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery query) { - String sql = buildProfileEntityRelationPathSql(query); - Object[] params = buildProfileEntityRelationPathParams(query); - - log.trace("[{}] profile entity relation path query: {}", tenantId, sql); - - return jdbcTemplate.queryForList(sql, params).stream() - .map(row -> { - var entityRelation = new EntityRelation(); - var fromId = (UUID) row.get(RELATION_FROM_ID_PROPERTY); - var fromType = (String) row.get(RELATION_FROM_TYPE_PROPERTY); - var toId = (UUID) row.get(RELATION_TO_ID_PROPERTY); - var toType = (String) row.get(RELATION_TO_TYPE_PROPERTY); - var grp = (String) row.get(RELATION_TYPE_GROUP_PROPERTY); - var type = (String) row.get(RELATION_TYPE_PROPERTY); - var version = (Long) row.get(VERSION_COLUMN); - - entityRelation.setFrom(EntityIdFactory.getByTypeAndUuid(fromType, fromId)); - entityRelation.setTo(EntityIdFactory.getByTypeAndUuid(toType, toId)); - entityRelation.setType(type); - entityRelation.setTypeGroup(RelationTypeGroup.valueOf(grp)); - entityRelation.setVersion(version); - return entityRelation; - }) - .collect(Collectors.toList()); - } - - private Object[] buildProfileEntityRelationPathParams(ProfileEntityRelationPathQuery query) { - final List params = new ArrayList<>(); - - params.add(query.rootEntityId().getId()); - params.add(query.rootEntityId().getEntityType().name()); - - params.add(query.level().relationType()); - - if (query.targetEntityProfileId() != null) { - params.add(query.targetEntityProfileId().getId()); - params.add(query.targetEntityProfileId().getId()); - } - - return params.toArray(); - } - - private static String buildProfileEntityRelationPathSql(ProfileEntityRelationPathQuery query) { - EntitySearchDirection direction = query.level().direction(); - - StringBuilder sb = new StringBuilder(); - - sb.append("\n") - .append("SELECT r.from_id, r.from_type, r.to_id, r.to_type,\n") - .append(" r.relation_type_group, r.relation_type, r.version\n") - .append("FROM ").append(RELATION_TABLE_NAME).append(" r\n"); - - sb.append("JOIN ").append(DEVICE_TABLE_NAME).append(" d ON "); - if (EntitySearchDirection.FROM == direction) { - sb.append("r.to_id = d.id AND r.to_type = 'DEVICE'").append("\n"); - } else { - sb.append("r.from_id = d.id AND r.from_type = 'DEVICE'").append("\n"); - } - - sb.append("JOIN ").append(ASSET_TABLE_NAME).append(" a ON "); - if (EntitySearchDirection.FROM == direction) { - sb.append("r.to_id = a.id AND r.to_type = 'ASSET'").append("\n"); - } else { - sb.append("r.from_id = a.id AND r.from_type = 'ASSET'").append("\n"); - } - - if (EntitySearchDirection.FROM == direction) { - sb.append("WHERE r.from_id = ?").append("\n") - .append("AND r.from_type = ?").append("\n"); - } else { - sb.append("WHERE r.to_id = ?").append("\n") - .append("AND r.to_type = ?").append("\n"); - } - - sb.append("AND r.relation_type = ?").append("\n") - .append("AND r.relation_type_group = '").append(RelationTypeGroup.COMMON).append("'\n"); - - if (query.targetEntityProfileId() != null) { - sb.append("AND ((d.device_profile_id = ?) OR (a.asset_profile_id = ?))").append("\n"); - } - - sb.append("AND (d.id IS NOT NULL OR a.id IS NOT NULL)"); - - return sb.toString(); - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index 0ebd5b6ceb..4b879f9d95 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -27,7 +27,6 @@ import org.thingsboard.server.dao.model.sql.RelationCompositeKey; import org.thingsboard.server.dao.model.sql.RelationEntity; import java.util.List; -import java.util.Optional; import java.util.UUID; public interface RelationRepository @@ -97,39 +96,4 @@ public interface RelationRepository @Param("toType") String toType, @Param("batchSize") int batchSize); - @Query(value = """ - SELECT r.from_id, r.from_type, r.relation_type_group, r.relation_type, r.to_id, r.to_type, r.additional_info, r.version - FROM relation r - LEFT JOIN device d ON r.to_id = d.id AND r.to_type = 'DEVICE' - LEFT JOIN asset a ON r.to_id = a.id AND r.to_type = 'ASSET' - WHERE r.from_id = :fromId - AND r.from_type = :fromType - AND r.relation_type = :relationType - AND r.relation_type_group = :relationTypeGroup - AND ((d.device_profile_id = :profileId) OR (a.asset_profile_id = :profileId)) - AND (d.id IS NOT NULL OR a.id IS NOT NULL) - """, nativeQuery = true) - List findByFromAndProfile(@Param("fromId") UUID fromId, - @Param("fromType") String fromType, - @Param("relationTypeGroup") String relationTypeGroup, - @Param("relationType") String relationType, - @Param("profileId") UUID profileId); - - @Query(value = """ - SELECT r.from_id, r.from_type, r.relation_type_group, r.relation_type, r.to_id, r.to_type, r.additional_info, r.version - FROM relation r - LEFT JOIN device d ON r.from_id = d.id AND r.from_type = 'DEVICE' - LEFT JOIN asset a ON r.from_id = a.id AND r.from_type = 'ASSET' - WHERE r.to_id = :toId - AND r.to_type = :toType - AND r.relation_type = :relationType - AND r.relation_type_group = :relationTypeGroup - AND ((d.device_profile_id = :profileId) OR (a.asset_profile_id = :profileId)) - AND (d.id IS NOT NULL OR a.id IS NOT NULL) - """, nativeQuery = true) - List findByToAndProfile(@Param("toId") UUID toId, - @Param("toType") String toType, - @Param("relationTypeGroup") String relationTypeGroup, - @Param("relationType") String relationType, - @Param("profileId") UUID profileId); } From 25ac2f2b487c48723b0f7d4c6ae8837b6b6de14e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 16 Oct 2025 16:32:35 +0300 Subject: [PATCH 07/33] added more tests and changed state proto --- .../cf/ctx/state/CalculatedFieldCtx.java | 15 +- ...ValuesAggregationCalculatedFieldState.java | 5 +- .../service/cf/ctx/state/aggregation/agg.json | 65 ------ .../server/utils/CalculatedFieldUtils.java | 4 + ...tValuesAggregationCalculatedFieldTest.java | 188 +++++++++++++++++- common/proto/src/main/proto/queue.proto | 1 + 6 files changed, 196 insertions(+), 82 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 3a9d75b3d4..f69d611b64 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -581,8 +581,7 @@ public class CalculatedFieldCtx { } if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig && other.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig - && thisConfig.getDeduplicationIntervalMillis() != otherConfig.getDeduplicationIntervalMillis() - && !thisConfig.getMetrics().equals(otherConfig.getMetrics())) { + && (thisConfig.getDeduplicationIntervalMillis() != otherConfig.getDeduplicationIntervalMillis() || !thisConfig.getMetrics().equals(otherConfig.getMetrics()))) { return true; } return false; @@ -596,7 +595,7 @@ public class CalculatedFieldCtx { var thisConfig = (AlarmCalculatedFieldConfiguration) calculatedField.getConfiguration(); var otherConfig = (AlarmCalculatedFieldConfiguration) other.getCalculatedField().getConfiguration(); if (!thisConfig.getCreateRules().equals(otherConfig.getCreateRules()) || - !Objects.equals(thisConfig.getClearRule(), otherConfig.getClearRule())) { + !Objects.equals(thisConfig.getClearRule(), otherConfig.getClearRule())) { return true; } } @@ -611,7 +610,7 @@ public class CalculatedFieldCtx { private boolean hasGeofencingZoneGroupConfigurationChanges(CalculatedFieldCtx other) { if (calculatedField.getConfiguration() instanceof GeofencingCalculatedFieldConfiguration thisConfig - && other.calculatedField.getConfiguration() instanceof GeofencingCalculatedFieldConfiguration otherConfig) { + && other.calculatedField.getConfiguration() instanceof GeofencingCalculatedFieldConfiguration otherConfig) { return !thisConfig.getZoneGroups().equals(otherConfig.getZoneGroups()); } return false; @@ -663,10 +662,10 @@ public class CalculatedFieldCtx { @Override public String toString() { return "CalculatedFieldCtx{" + - "cfId=" + cfId + - ", cfType=" + cfType + - ", entityId=" + entityId + - '}'; + "cfId=" + cfId + + ", cfType=" + cfType + + ", entityId=" + entityId + + '}'; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java index 1df27b4cbd..6ae023ed1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java @@ -48,6 +48,7 @@ import java.util.Map.Entry; @Getter public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedFieldState { + @Setter private long lastArgsRefreshTs = -1; @Setter private long lastMetricsEvalTs = -1; @@ -74,6 +75,7 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF lastArgsRefreshTs = -1; lastMetricsEvalTs = -1; metrics = null; + inputs.clear(); } @Override @@ -104,7 +106,8 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception { - if (!shouldRecalculate()) { + boolean shouldRecalculate = updatedArgs == null || updatedArgs.isEmpty(); + if (!shouldRecalculate() && !shouldRecalculate) { return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() .result(null) .build()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json deleted file mode 100644 index b39092ca74..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "type": "LATEST_VALUES_AGGREGATION", - "name": "Occupied spaces", - "debugSettings": { - "failuresEnabled": true, - "allEnabled": true, - "allEnabledUntil": 1769907492297 - }, - "entityId": { - "entityType": "ASSET_PROFILE", - "id": "bb8ddd40-a8bc-11f0-869b-e9d81fa6eaf1" - }, - "configuration": { - "type": "LATEST_VALUES_AGGREGATION", - "source": { - "relation": { - "direction": "FROM", - "relationType": "Contains" - }, - "entityProfiles": [ - { - "entityType": "DEVICE_PROFILE", - "id": "d7a05580-a4cf-11f0-87cb-2d6683c4fccf" - } - ] - }, - "inputs": { - "oc": { - "key": "occupied", - "type": "TS_LATEST" - } - }, - "deduplicationIntervalMillis": 10000, - "metrics": { - "totalSpaces": { - "function": "COUNT", - "input": { - "type": "function", - "function" : "return 1;" - } - }, - "occupiedSpaces": { - "function": "COUNT", - "filter": "return oc == true", - "input": { - "type": "key", - "key" : "oc" - } - }, - "freeSpaces": { - "function": "COUNT", - "filter": "return oc == false", - "input": { - "type": "key", - "key" : "oc" - } - } - }, - "output": { - "type": "TIME_SERIES", - "decimals": 2 - }, - "useLatestTsFromInputs": "true" - } -} diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 19ad4bfb7c..00f4cbde0f 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -96,6 +96,9 @@ public class CalculatedFieldUtils { .setId(toProto(stateId)) .setType(state.getType().name()); + if (state instanceof LatestValuesAggregationCalculatedFieldState aggState) { + builder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); + } state.getArguments().forEach((argName, argEntry) -> { if (argEntry instanceof AggArgumentEntry aggArgumentEntry) { aggArgumentEntry.getAggInputs() @@ -242,6 +245,7 @@ public class CalculatedFieldUtils { arguments.forEach((argName, entityInputs) -> { aggState.getArguments().put(argName, new AggArgumentEntry(entityInputs, false)); }); + aggState.setLastArgsRefreshTs(proto.getLastArgsUpdateTs()); } } diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java index a4b43e362b..8c2c2ca68c 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -15,10 +15,13 @@ */ package org.thingsboard.server.cf; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.Tenant; @@ -61,6 +64,7 @@ import static org.awaitility.Awaitility.await; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.cf.CalculatedFieldIntegrationTest.POLL_INTERVAL; +@Slf4j @DaoSqlTest public class LatestValuesAggregationCalculatedFieldTest extends AbstractControllerTest { @@ -341,13 +345,17 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset2.getId(), device3.getId(), "Contains"); createEntityRelation(asset2.getId(), device4.getId(), "Contains"); - postTelemetry(device3.getId(), "{\"occupied\":false}"); - postTelemetry(device4.getId(), "{\"occupied\":true}"); - postTelemetry(device3.getId(), "{\"occupied\":true}"); + long currentTime = System.currentTimeMillis(); + long firstTs = currentTime - 10; + long secondTs = currentTime - 10; + long thirdTs = currentTime - 5; + postTelemetry(device3.getId(), "{\"ts\": " + firstTs + ", \"values\": {\"occupied\":true}}"); + postTelemetry(device4.getId(), "{\"ts\": " + secondTs + ", \"values\": {\"occupied\":true}}"); + postTelemetry(device3.getId(), "{\"ts\": " + thirdTs + ", \"values\": {\"occupied\":true}}"); createOccupancyCF(asset2.getId()); - await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -357,15 +365,15 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll )); }); - doDelete("/api/plugins/telemetry/DEVICE/" + device3.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=0&endTs=" + System.currentTimeMillis(), String.class); - doDelete("/api/plugins/telemetry/DEVICE/" + device4.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=0&endTs=" + System.currentTimeMillis(), String.class); + doDelete("/api/plugins/telemetry/DEVICE/" + device3.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=" + thirdTs + "&endTs=" + thirdTs + 1, String.class); + doDelete("/api/plugins/telemetry/DEVICE/" + device4.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=" + secondTs + "&endTs=" + secondTs + 1, String.class); await().alias("delete latest telemetry and perform aggregation with previous or default values").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( - "freeSpaces", "2", - "occupiedSpaces", "0", + "freeSpaces", "1", + "occupiedSpaces", "1", "totalSpaces", "2" )); }); @@ -411,6 +419,140 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll }); } + @Test + public void testUpdateRelationPath_checkAggregation() throws Exception { + CalculatedField cf = createOccupancyCF(asset.getId()); + checkInitialCalculation(); + + Device device3 = createDevice("Device 3", "1234567890333"); + createEntityRelation(asset.getId(), device3.getId(), "Has"); + postTelemetry(device3.getId(), "{\"occupied\":true}"); + + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + configuration.setRelation(new RelationPathLevel(EntitySearchDirection.FROM, "Has")); + saveCalculatedField(cf); + + await().alias("update relation path and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "0", + "occupiedSpaces", "1", + "totalSpaces", "1" + )); + }); + } + + @Test + public void testUpdateArguments_checkAggregation() throws Exception { + CalculatedField cf = createOccupancyCF(asset.getId()); + checkInitialCalculation(); + + postTelemetry(device1.getId(), "{\"occupiedStatus\":false}"); + postTelemetry(device2.getId(), "{\"occupiedStatus\":false}"); + + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + Argument argument = new Argument(); + argument.setRefEntityKey(new ReferencedEntityKey("oc", ArgumentType.TS_LATEST, null)); + argument.setDefaultValue("false"); + configuration.setArguments(Map.of("oc", argument)); + saveCalculatedField(cf); + + await().alias("update arguments and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); + }); + } + + @Test + public void testUpdateMetrics_checkAggregation() throws Exception { + postTelemetry(device1.getId(), "{\"temperature\":24.2}"); + postTelemetry(device2.getId(), "{\"temperature\":19.6}"); + CalculatedField cf = createAvgTemperatureCF(asset.getId()); + + await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); + }); + + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + AggMetric aggMetric = new AggMetric(); + aggMetric.setInput(new AggKeyInput("temp")); + aggMetric.setFilter("return temp < 100;"); + aggMetric.setFunction(AggFunction.MAX); + configuration.setMetrics(Map.of("maxTemperature", aggMetric)); + saveCalculatedField(cf); + + postTelemetry(device1.getId(), "{\"temperature\":101.3}"); + postTelemetry(device2.getId(), "{\"temperature\":25.8}"); + + await().alias("update metrics and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("maxTemperature", "26")); + }); + } + + @Test + public void testUpdateOutput_checkAggregation() throws Exception { + postTelemetry(device1.getId(), "{\"temperature\":24.2}"); + postTelemetry(device2.getId(), "{\"temperature\":19.6}"); + CalculatedField cf = createAvgTemperatureCF(asset.getId()); + + await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); + }); + + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + Output output = new Output(); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + configuration.setOutput(output); + saveCalculatedField(cf); + + await().alias("update output and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode avgTemperature = getServerAttributes(asset.getId(), "avgTemperature"); + assertThat(avgTemperature).isNotNull(); + assertThat(avgTemperature.get(0)).isNotNull(); + assertThat(avgTemperature.get(0).get("value").asText()).isEqualTo("24.2"); + }); + } + + @Test + public void testUpdateDeduplicationInterval_checkAggregationNotExecutedUntilDeduplicationInterval() throws Exception { + postTelemetry(device1.getId(), "{\"temperature\":24.2}"); + postTelemetry(device2.getId(), "{\"temperature\":19.6}"); + CalculatedField cf = createAvgTemperatureCF(asset.getId()); + + await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); + }); + + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + configuration.setDeduplicationIntervalMillis(2 * deduplicationInterval); + saveCalculatedField(cf); + + postTelemetry(device2.getId(), "{\"temperature\":32.1}"); + + await().alias("update deduplication interval and perform aggregation").atMost(2 * deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("avgTemperature", "28")); + }); + } + private void checkInitialCalculation() { await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) @@ -425,6 +567,32 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); } + private CalculatedField createAvgTemperatureCF(EntityId entityId) { + Map arguments = new HashMap<>(); + Argument argument = new Argument(); + argument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + argument.setDefaultValue("20"); + arguments.put("temp", argument); + + Map aggMetrics = new HashMap<>(); + + AggMetric avgMetric = new AggMetric(); + avgMetric.setFunction(AggFunction.AVG); + avgMetric.setFilter("return temp >= 20;"); + avgMetric.setInput(new AggKeyInput("temp")); + aggMetrics.put("avgTemperature", avgMetric); + + Output output = new Output(); + output.setType(OutputType.TIME_SERIES); + output.setDecimalsByDefault(0); + + return createAggCf("Average temperature", entityId, + new RelationPathLevel(EntitySearchDirection.FROM, "Contains"), + arguments, + aggMetrics, + output); + } + private CalculatedField createOccupancyCF(EntityId entityId) { Map arguments = new HashMap<>(); Argument argument = new Argument(); @@ -513,4 +681,8 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); } + private ArrayNode getServerAttributes(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/attributes/SERVER_SCOPE?keys=" + String.join(",", keys), ArrayNode.class); + } + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 764f4c3ec9..1e01916e55 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -928,6 +928,7 @@ message CalculatedFieldStateProto { repeated GeofencingArgumentProto geofencingArguments = 5; AlarmStateProto alarmState = 6; repeated AggSingleArgumentEntryProto aggArguments = 7; + int64 lastArgsUpdateTs = 8; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. From f951c64767c504142bdaaab1ad69711d995ec73c Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 17 Oct 2025 11:14:07 +0300 Subject: [PATCH 08/33] added state to proto and fixed deduplication logic --- ...ValuesAggregationCalculatedFieldState.java | 23 +++---- .../state/aggregation/function/new_agg.json | 60 ------------------- .../server/utils/CalculatedFieldUtils.java | 16 +++-- ...tValuesAggregationCalculatedFieldTest.java | 16 ++++- common/proto/src/main/proto/queue.proto | 8 ++- 5 files changed, 42 insertions(+), 81 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java index 6ae023ed1e..f3560faafa 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java @@ -106,21 +106,22 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception { - boolean shouldRecalculate = updatedArgs == null || updatedArgs.isEmpty(); - if (!shouldRecalculate() && !shouldRecalculate) { + boolean cfUpdated = updatedArgs != null && updatedArgs.isEmpty(); + if (shouldRecalculate() || cfUpdated) { + Output output = ctx.getOutput(); + ObjectNode aggResult = aggregateMetrics(output); + lastMetricsEvalTs = System.currentTimeMillis(); + ctx.scheduleReevaluation(deduplicationInterval, actorCtx); + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .type(output.getType()) + .scope(output.getScope()) + .result(createResultJson(ctx.isUseLatestTs(), aggResult)) + .build()); + } else { return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() .result(null) .build()); } - Output output = ctx.getOutput(); - ObjectNode aggResult = aggregateMetrics(output); - lastMetricsEvalTs = System.currentTimeMillis(); - ctx.scheduleReevaluation(deduplicationInterval, actorCtx); - return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() - .type(output.getType()) - .scope(output.getScope()) - .result(createResultJson(ctx.isUseLatestTs(), aggResult)) - .build()); } private boolean shouldRecalculate() { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json deleted file mode 100644 index c6b841b673..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "type": "LATEST_VALUES_AGGREGATION", - "name": "Occupied spaces", - "debugSettings": { - "failuresEnabled": true, - "allEnabled": true, - "allEnabledUntil": 1769907492297 - }, - "entityId": { - "entityType": "ASSET", - "id": "f8ad0800-a9a6-11f0-bbe6-459b63b420fe" - }, - "configuration": { - "type": "LATEST_VALUES_AGGREGATION", - "relation": { - "direction": "FROM", - "relationType": "Contains" - }, - "arguments": { - "oc": { - "refEntityKey": { - "key": "occupied", - "type": "TS_LATEST" - }, - "defaultValue": "false" - } - }, - "deduplicationIntervalMillis": 20000, - "metrics": { - "totalSpaces": { - "function": "COUNT", - "input": { - "type": "function", - "function" : "return 1;" - } - }, - "occupiedSpaces": { - "function": "COUNT", - "filter": "return oc == true", - "input": { - "type": "key", - "key" : "oc" - } - }, - "freeSpaces": { - "function": "COUNT", - "filter": "return oc == false", - "input": { - "type": "key", - "key" : "oc" - } - } - }, - "output": { - "type": "TIME_SERIES", - "decimals": 2 - }, - "useLatestTsFromInputs": "true" - } -} diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 00f4cbde0f..f8a7d17543 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -35,6 +35,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdPro import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingZoneProto; +import org.thingsboard.server.gen.transport.TransportProtos.LatestValuesAggregationStateProto; import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.TsDoubleValProto; import org.thingsboard.server.gen.transport.TransportProtos.TsRollingArgumentProto; @@ -96,13 +97,11 @@ public class CalculatedFieldUtils { .setId(toProto(stateId)) .setType(state.getType().name()); - if (state instanceof LatestValuesAggregationCalculatedFieldState aggState) { - builder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); - } + LatestValuesAggregationStateProto.Builder aggBuilder = LatestValuesAggregationStateProto.newBuilder(); state.getArguments().forEach((argName, argEntry) -> { if (argEntry instanceof AggArgumentEntry aggArgumentEntry) { aggArgumentEntry.getAggInputs() - .forEach((entityId, entry) -> builder.addAggArguments(toAggSingleArgumentProto(argName, entityId, entry))); + .forEach((entityId, entry) -> aggBuilder.addAggArguments(toAggSingleArgumentProto(argName, entityId, entry))); } else if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { @@ -120,6 +119,10 @@ public class CalculatedFieldUtils { alarmStateProto.setClearRuleState(toAlarmRuleStateProto(alarmState.getClearRuleState())); } } + if (state instanceof LatestValuesAggregationCalculatedFieldState aggState) { + aggBuilder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); + builder.setLatestValuesAggregationState(aggBuilder.build()); + } return builder.build(); } @@ -237,15 +240,16 @@ public class CalculatedFieldUtils { } case LATEST_VALUES_AGGREGATION -> { LatestValuesAggregationCalculatedFieldState aggState = (LatestValuesAggregationCalculatedFieldState) state; + LatestValuesAggregationStateProto aggregationStateProto = proto.getLatestValuesAggregationState(); Map> arguments = new HashMap<>(); - proto.getAggArgumentsList().forEach(argProto -> { + aggregationStateProto.getAggArgumentsList().forEach(argProto -> { AggSingleEntityArgumentEntry entry = fromAggSingleValueArgumentProto(argProto); arguments.computeIfAbsent(argProto.getValue().getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); }); arguments.forEach((argName, entityInputs) -> { aggState.getArguments().put(argName, new AggArgumentEntry(entityInputs, false)); }); - aggState.setLastArgsRefreshTs(proto.getLastArgsUpdateTs()); + aggState.setLastArgsRefreshTs(aggregationStateProto.getLastArgsUpdateTs()); } } diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java index 8c2c2ca68c..46b8b78d57 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -489,10 +489,16 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll configuration.setMetrics(Map.of("maxTemperature", aggMetric)); saveCalculatedField(cf); + await().alias("update metrics and perform aggregation").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("maxTemperature", "24")); + }); + postTelemetry(device1.getId(), "{\"temperature\":101.3}"); postTelemetry(device2.getId(), "{\"temperature\":25.8}"); - await().alias("update metrics and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("maxTemperature", "26")); @@ -544,9 +550,15 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll configuration.setDeduplicationIntervalMillis(2 * deduplicationInterval); saveCalculatedField(cf); + await().alias("update deduplication interval and perform aggregation").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); + }); + postTelemetry(device2.getId(), "{\"temperature\":32.1}"); - await().alias("update deduplication interval and perform aggregation").atMost(2 * deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(2 * deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("avgTemperature", "28")); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 1e01916e55..b59e01128b 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -920,6 +920,11 @@ message AggSingleArgumentEntryProto { SingleValueArgumentProto value = 2; } +message LatestValuesAggregationStateProto { + int64 lastArgsUpdateTs = 1; + repeated AggSingleArgumentEntryProto aggArguments = 2; +} + message CalculatedFieldStateProto { CalculatedFieldEntityCtxIdProto id = 1; string type = 2; @@ -927,8 +932,7 @@ message CalculatedFieldStateProto { repeated TsRollingArgumentProto rollingValueArguments = 4; repeated GeofencingArgumentProto geofencingArguments = 5; AlarmStateProto alarmState = 6; - repeated AggSingleArgumentEntryProto aggArguments = 7; - int64 lastArgsUpdateTs = 8; + LatestValuesAggregationStateProto latestValuesAggregationState = 7; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. From 6a307041b699f4d2a5d1c27e8755b8538f0fec31 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 17 Oct 2025 15:55:44 +0300 Subject: [PATCH 09/33] added tests for entry --- ...CalculatedFieldEntityMessageProcessor.java | 4 +- ...tractCalculatedFieldProcessingService.java | 4 +- .../DefaultCalculatedFieldQueueService.java | 24 ++-- .../ctx/state/BaseCalculatedFieldState.java | 3 +- .../state/aggregation/AggArgumentEntry.java | 7 +- .../AggSingleEntityArgumentEntry.java | 30 +++-- .../cf/ctx/state/AggArgumentEntryTest.java | 112 ++++++++++++++++++ .../AggSingleEntityArgumentEntryTest.java | 101 ++++++++++++++++ .../server/dao/relation/RelationService.java | 2 + .../data/cf/configuration/Argument.java | 4 + ...gregationCalculatedFieldConfiguration.java | 13 ++ .../dao/relation/BaseRelationService.java | 15 +++ 12 files changed, 288 insertions(+), 31 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index ce29dc06e0..58113deb08 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -67,7 +67,6 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -686,9 +685,8 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM Map.Entry::getKey, argEntry -> new AggSingleEntityArgumentEntry(entityId, argEntry.getValue()) )); - } else { - fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); } + fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); return fetchedArgs; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 94695bcc8c..7f47dba2ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -332,7 +332,7 @@ public abstract class AbstractCalculatedFieldProcessingService { var attributeOptFuture = attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()); return Futures.transform(attributeOptFuture, attrOpt -> { log.debug("[{}][{}] Fetched attribute for key {}: {}", tenantId, entityId, argument.getRefEntityKey(), attrOpt); - AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, 0L)); + AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, SingleValueArgumentEntry.DEFAULT_VERSION)); return transformAggSingleArgument(entityId, Optional.of(attributeKvEntry)); }, calculatedFieldCallbackExecutor); } @@ -344,7 +344,7 @@ public abstract class AbstractCalculatedFieldProcessingService { timeseriesService.findLatest(tenantId, entityId, key), result -> { log.debug("[{}][{}] Fetched latest timeseries {}: {}", tenantId, entityId, key, result); - Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(defaultTs, createDefaultKvEntry(argument), 0L))); + Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(defaultTs, createDefaultKvEntry(argument), SingleValueArgumentEntry.DEFAULT_VERSION))); return transformAggSingleArgument(entityId, tsKvEntry); }, calculatedFieldCallbackExecutor); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 37c7b2ef6c..147882c3c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -37,8 +37,9 @@ import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationPathLevel; -import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; @@ -191,20 +192,15 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS for (CalculatedFieldCtx cfCtx : cfCtxs) { if (cfCtx.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { RelationPathLevel relation = aggConfig.getRelation(); - switch (relation.direction()) { - case FROM -> { - List byToAndType = relationService.findByToAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); - if (!byToAndType.isEmpty()) { - return true; - } - } - case TO -> { - List byFromAndType = relationService.findByFromAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); - if (!byFromAndType.isEmpty()) { - return true; - } - } + EntitySearchDirection inverseDirection = switch (relation.direction()) { + case FROM -> EntitySearchDirection.TO; + case TO -> EntitySearchDirection.FROM; }; + RelationPathLevel inverseRelation = new RelationPathLevel(inverseDirection, relation.relationType()); + List byRelationPathQuery = relationService.findByRelationPathQuery(tenantId, new EntityRelationPathQuery(entityId, List.of(inverseRelation))); + if (!byRelationPathQuery.isEmpty()) { + return true; + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 2d853fd3fd..ee10857e6f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -22,6 +22,7 @@ import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.utils.CalculatedFieldUtils; import java.io.Closeable; @@ -73,7 +74,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, ArgumentEntry existingEntry = arguments.get(key); boolean entryUpdated; - if (existingEntry == null || newEntry.isForceResetPrevious()) { + if (existingEntry == null || !(newEntry instanceof AggSingleEntityArgumentEntry) && newEntry.isForceResetPrevious()) { validateNewEntry(key, newEntry); arguments.put(key, newEntry); entryUpdated = true; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java index 7e3a8623e4..e002aece2c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java @@ -52,7 +52,12 @@ public class AggArgumentEntry implements ArgumentEntry { if (aggSingleEntityArgumentEntry.isDeleted()) { aggInputs.remove(aggSingleEntityArgumentEntry.getEntityId()); } else { - aggInputs.put(aggSingleEntityArgumentEntry.getEntityId(), aggSingleEntityArgumentEntry); + ArgumentEntry argumentEntry = aggInputs.get(aggSingleEntityArgumentEntry.getEntityId()); + if (argumentEntry != null) { + argumentEntry.updateEntry(aggSingleEntityArgumentEntry); + } else { + aggInputs.put(aggSingleEntityArgumentEntry.getEntityId(), aggSingleEntityArgumentEntry); + } } return true; } else { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java index 32ce77311d..6430fe3f1c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java @@ -62,25 +62,35 @@ public class AggSingleEntityArgumentEntry extends SingleValueArgumentEntry { @Override public boolean updateEntry(ArgumentEntry entry) { - if (entry instanceof AggSingleEntityArgumentEntry singleValueEntry) { - if (singleValueEntry.getTs() <= ts) { - return false; + if (entry instanceof AggSingleEntityArgumentEntry aggSingleEntityEntry) { + if (aggSingleEntityEntry.isForceResetPrevious()) { + return applyNewEntry(aggSingleEntityEntry); } - Long newVersion = singleValueEntry.getVersion(); + if (aggSingleEntityEntry.getTs() < this.ts) { + if (!isDefaultValue()) { + return false; + } + } + + Long newVersion = aggSingleEntityEntry.getVersion(); if (newVersion == null || this.version == null || newVersion > this.version) { - this.ts = singleValueEntry.getTs(); - this.version = newVersion; - this.kvEntryValue = singleValueEntry.getKvEntryValue(); - this.entityId = singleValueEntry.getEntityId(); - return true; + return applyNewEntry(aggSingleEntityEntry); } } else { - throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); + throw new IllegalArgumentException("Unsupported argument entry type for aggregation single entity argument entry: " + entry.getType()); } return false; } + private boolean applyNewEntry(AggSingleEntityArgumentEntry entry) { + this.ts = entry.getTs(); + this.version = entry.getVersion(); + this.kvEntryValue = entry.getKvEntryValue(); + this.entityId = entry.getEntityId(); + return true; + } + @Override public ArgumentEntryType getType() { return ArgumentEntryType.AGGREGATE_LATEST_SINGLE; diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java new file mode 100644 index 0000000000..c1c2d85c55 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java @@ -0,0 +1,112 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class AggArgumentEntryTest { + + private AggArgumentEntry entry; + + private final DeviceId device1 = new DeviceId(UUID.fromString("1984e5f4-9ff0-4187-84ae-e4438bba4c8a")); + private final DeviceId device2 = new DeviceId(UUID.fromString("937fc062-1a9d-438f-aa22-55a93fc908b7")); + + private final long ts = System.currentTimeMillis(); + + @BeforeEach + void setUp() { + Map aggInputs = new HashMap<>(); + aggInputs.put(device1, new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 12L), 1L))); + aggInputs.put(device2, new AggSingleEntityArgumentEntry(device2, new BasicTsKvEntry(ts - 150, new LongDataEntry("key", 16L), 6L))); + + entry = new AggArgumentEntry(aggInputs, false); + } + + @Test + void testUpdateEntryWhenNotAggEntryPassed() { + assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported argument entry type for aggregation argument entry: " + ArgumentEntryType.TS_ROLLING); + } + + @Test + void testUpdateEntryWhenAggArgumentEntryPasser() { + DeviceId device3 = new DeviceId(UUID.randomUUID()); + DeviceId device4 = new DeviceId(UUID.randomUUID()); + + AggArgumentEntry aggArgumentEntry = new AggArgumentEntry(Map.of( + device3, new AggSingleEntityArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 16L), 13L)), + device4, new AggSingleEntityArgumentEntry(device4, new BasicTsKvEntry(ts - 60, new LongDataEntry("key", 23L), 7L)) + ), false); + + assertThat(entry.updateEntry(aggArgumentEntry)).isTrue(); + + Map aggInputs = entry.getAggInputs(); + assertThat(aggInputs.size()).isEqualTo(4); + assertThat(aggInputs.get(device3)).isEqualTo(aggArgumentEntry.getAggInputs().get(device3)); + assertThat(aggInputs.get(device4)).isEqualTo(aggArgumentEntry.getAggInputs().get(device4)); + } + + @Test + void testUpdateEntryWhenAggSingleEntityArgumentEntryPassedAndNoEntriesById() { + DeviceId device3 = new DeviceId(UUID.randomUUID()); + + AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 10L)); + + assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); + + Map aggInputs = entry.getAggInputs(); + assertThat(aggInputs.size()).isEqualTo(3); + assertThat(aggInputs.get(device3)).isEqualTo(singleEntityArgumentEntry); + } + + @Test + void testUpdateEntryWhenAggSingleEntityArgumentEntryPassedAndEntryByIdExist() { + AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device2, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 10L)); + + assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); + + Map aggInputs = entry.getAggInputs(); + assertThat(aggInputs.size()).isEqualTo(2); + assertThat(aggInputs.get(device2)).isEqualTo(singleEntityArgumentEntry); + } + + @Test + void testUpdateEntryWhenDeletedAggSingleEntityArgumentEntryPassed() { + AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device2, true); + + assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); + + Map aggInputs = entry.getAggInputs(); + assertThat(aggInputs.size()).isEqualTo(1); + assertThat(aggInputs.get(device2)).isNull(); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java new file mode 100644 index 0000000000..0401bb0156 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java @@ -0,0 +1,101 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class AggSingleEntityArgumentEntryTest { + + private AggSingleEntityArgumentEntry entry; + + private final DeviceId device1 = new DeviceId(UUID.fromString("1984e5f4-9ff0-4187-84ae-e4438bba4c8a")); + + private final long ts = System.currentTimeMillis(); + + @BeforeEach + void setUp() { + entry = new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 12L), 22L)); + } + + @Test + void testUpdateEntryWhenNotAggEntryPassed() { + assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported argument entry type for aggregation single entity argument entry: " + ArgumentEntryType.TS_ROLLING); + } + + @Test + void testUpdateEntryWhenResetPrevious() { + AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 100L)); + singleEntityArgumentEntry.setForceResetPrevious(true); + + assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); + assertThat(entry.getTs()).isEqualTo(singleEntityArgumentEntry.getTs()); + assertThat(entry.getKvEntryValue()).isEqualTo(singleEntityArgumentEntry.getKvEntryValue()); + assertThat(entry.getVersion()).isEqualTo(singleEntityArgumentEntry.getVersion()); + } + + + @Test + void testUpdateEntryWithTheSameTsAndVersion() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 19L), 22L)))).isFalse(); + } + + @Test + void testUpdateEntryWithTheSameTsAndDifferentVersion() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 134L), 23L)))).isTrue(); + } + + @Test + void testUpdateEntryWhenNewVersionIsNull() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 56L), null)))).isTrue(); + assertThat(entry.getValue()).isEqualTo(56L); + assertThat(entry.getVersion()).isNull(); + } + + @Test + void testUpdateEntryWhenNewVersionIsGreaterThanCurrent() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 76L), 23L)))).isTrue(); + assertThat(entry.getValue()).isEqualTo(76L); + assertThat(entry.getVersion()).isEqualTo(23); + } + + @Test + void testUpdateEntryWhenNewVersionIsLessThanCurrent() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 11L), 20L)))).isFalse(); + } + + @Test + void testUpdateEntryWhenValueWasNotChanged() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 18L), 45L)))).isTrue(); + } + + @Test + void testUpdateEntryWithOldTs() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 150, new LongDataEntry("key", 155L), 45L)))).isFalse(); + } + +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index a0bc9a72e6..214dc5247e 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -86,6 +86,8 @@ public interface RelationService { ListenableFuture> findByRelationPathQueryAsync(TenantId tenantId, EntityRelationPathQuery relationPathQuery); + List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery relationPathQuery); + // TODO: This method may be useful for some validations in the future // ListenableFuture checkRecursiveRelation(EntityId from, EntityId to); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index 4de4551e73..8d4a831cf9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -45,4 +45,8 @@ public class Argument { return hasDynamicSource() && refDynamicSourceConfiguration.getType() == CFArgumentDynamicSourceType.CURRENT_OWNER; } + public boolean hasTsRollingArgument() { + return ArgumentType.TS_ROLLING.equals(refEntityKey.getType()); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java index 43a3360ce6..721930e676 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -41,6 +41,19 @@ public class LatestValuesAggregationCalculatedFieldConfiguration implements Argu @Override public void validate() { + if (relation == null) { + throw new IllegalArgumentException("Relation must be specified!"); + } + relation.validate(); + if (arguments.containsKey("ctx")) { + throw new IllegalArgumentException("Argument name 'ctx' is reserved and cannot be used."); + } + if (arguments.values().stream().anyMatch(Argument::hasTsRollingArgument)) { + throw new IllegalArgumentException("Calculated field with type: '" + getType() + "' doesn't support TS_ROLLING arguments."); + } + if (metrics.isEmpty()) { + throw new IllegalArgumentException("Latest value aggregation calculated field must have at least one metric."); + } } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 18d1806fe4..ec203e5309 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -514,6 +514,21 @@ public class BaseRelationService implements RelationService { return executor.submit(() -> relationDao.findByRelationPathQuery(tenantId, relationPathQuery)); } + @Override + public List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery relationPathQuery) { + log.trace("Executing findByRelationPathQuery, tenantId [{}], relationPathQuery {}", tenantId, relationPathQuery); + validateId(tenantId, id -> "Invalid tenant id: " + id); + validate(relationPathQuery); + if (relationPathQuery.levels().size() == 1) { + RelationPathLevel relationPathLevel = relationPathQuery.levels().get(0); + return switch (relationPathLevel.direction()) { + case FROM -> findByFromAndType(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), RelationTypeGroup.COMMON); + case TO -> findByToAndType(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), RelationTypeGroup.COMMON); + }; + } + return relationDao.findByRelationPathQuery(tenantId, relationPathQuery); + } + private void validate(EntityRelationPathQuery relationPathQuery) { validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); List levels = relationPathQuery.levels(); From 37039a995dca6c3fa05395992ce335d4b579083a Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 20 Oct 2025 10:02:55 +0300 Subject: [PATCH 10/33] added minDeduplicationInterval to tenant profile config --- .../main/data/upgrade/basic/schema_update.sql | 8 +++ .../controller/SystemInfoController.java | 1 + .../cf/ctx/state/CalculatedFieldCtx.java | 2 +- ...ValuesAggregationCalculatedFieldState.java | 2 +- ...tValuesAggregationCalculatedFieldTest.java | 64 ++++++++++--------- .../server/common/data/SystemParams.java | 1 + ...gregationCalculatedFieldConfiguration.java | 2 +- .../DefaultTenantProfileConfiguration.java | 2 + .../CalculatedFieldDataValidator.java | 15 ++++- 9 files changed, 63 insertions(+), 34 deletions(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 495aee00e2..c5b73c6899 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -34,6 +34,12 @@ SET profile_data = jsonb_set( WHEN (profile_data -> 'configuration') ? 'maxRelationLevelPerCfArgument' THEN NULL ELSE to_jsonb(10) + END, + 'minAllowedDeduplicationIntervalInSecForCF', + CASE + WHEN (profile_data -> 'configuration') ? 'minAllowedDeduplicationIntervalInSecForCF' + THEN NULL + ELSE to_jsonb(3600) END ) ), @@ -43,6 +49,8 @@ WHERE NOT ( (profile_data -> 'configuration') ? 'minAllowedScheduledUpdateIntervalInSecForCF' AND (profile_data -> 'configuration') ? 'maxRelationLevelPerCfArgument' + AND + (profile_data -> 'configuration') ? 'minAllowedDeduplicationIntervalInSecForCF' ); -- UPDATE TENANT PROFILE CONFIGURATION END diff --git a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java index b9968aefa9..82807d0762 100644 --- a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java +++ b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java @@ -164,6 +164,7 @@ public class SystemInfoController extends BaseController { systemParams.setMaxDataPointsPerRollingArg(tenantProfileConfiguration.getMaxDataPointsPerRollingArg()); systemParams.setMinAllowedScheduledUpdateIntervalInSecForCF(tenantProfileConfiguration.getMinAllowedScheduledUpdateIntervalInSecForCF()); systemParams.setMaxRelationLevelPerCfArgument(tenantProfileConfiguration.getMaxRelationLevelPerCfArgument()); + systemParams.setMinAllowedDeduplicationIntervalInSecForCF(tenantProfileConfiguration.getMinAllowedDeduplicationIntervalInSecForCF()); systemParams.setTrendzSettings(trendzSettingsService.findTrendzSettings(currentUser.getTenantId())); } systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID)) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index f69d611b64..1d9ccf10d3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -581,7 +581,7 @@ public class CalculatedFieldCtx { } if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig && other.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig - && (thisConfig.getDeduplicationIntervalMillis() != otherConfig.getDeduplicationIntervalMillis() || !thisConfig.getMetrics().equals(otherConfig.getMetrics()))) { + && (thisConfig.getDeduplicationIntervalInSec() != otherConfig.getDeduplicationIntervalInSec() || !thisConfig.getMetrics().equals(otherConfig.getMetrics()))) { return true; } return false; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java index f3560faafa..fdc045a0db 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java @@ -66,7 +66,7 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF super.setCtx(ctx, actorCtx); var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); metrics = configuration.getMetrics(); - deduplicationInterval = configuration.getDeduplicationIntervalMillis(); + deduplicationInterval = configuration.getDeduplicationIntervalInSec(); } @Override diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java index 46b8b78d57..e8c244b57e 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -79,12 +79,16 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll private AssetProfile assetProfile; private Asset asset; - private long deduplicationInterval = 10000; + private long deduplicationInterval = 10; @Before public void beforeTest() throws Exception { loginSysAdmin(); + updateDefaultTenantProfileConfig(tenantProfileConfig -> { + tenantProfileConfig.setMinAllowedDeduplicationIntervalInSecForCF(1); + }); + Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); savedTenant = saveTenant(tenant); @@ -131,7 +135,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createOccupancyCF(assetProfile.getId()); - await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -149,7 +153,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device3.getId(), "{\"occupied\":true}"); - await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -171,7 +175,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll Asset asset2 = createAsset("Asset 2", assetProfile.getId()); - await().alias("add entity to profile with no related entities and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("add entity to profile with no related entities and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode occupancy = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); @@ -184,7 +188,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset2.getId(), device3.getId(), "Contains"); createEntityRelation(asset2.getId(), device4.getId(), "Contains"); - await().alias("create relations and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create relations and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -196,7 +200,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device3.getId(), "{\"occupied\":false}"); - await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval * 2, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -218,7 +222,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createOccupancyCF(assetProfile.getId()); - await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -240,7 +244,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device3.getId(), "{\"occupied\":true}"); - await().alias("change profile and no aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("change profile and no aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -262,7 +266,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createOccupancyCF(asset2.getId()); - await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -280,7 +284,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device1.getId(), "{\"occupied\":false}"); - await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -301,7 +305,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device1.getId(), "{\"occupied\":false}"); - await().alias("delete cf and update telemetry and no aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("delete cf and update telemetry and no aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -319,13 +323,13 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device1.getId(), "{\"occupied\":false}"); - await().alias("update telemetry -> no changes").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + await().alias("update telemetry -> no changes").atMost(deduplicationInterval / 2, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(this::checkInitialCalculationValues); postTelemetry(device2.getId(), "{\"occupied\":false}"); - await().alias("create CF and perform initial calculation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform initial calculation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -355,7 +359,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createOccupancyCF(asset2.getId()); - await().alias("create CF and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -368,7 +372,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll doDelete("/api/plugins/telemetry/DEVICE/" + device3.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=" + thirdTs + "&endTs=" + thirdTs + 1, String.class); doDelete("/api/plugins/telemetry/DEVICE/" + device4.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=" + secondTs + "&endTs=" + secondTs + 1, String.class); - await().alias("delete latest telemetry and perform aggregation with previous or default values").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) + await().alias("delete latest telemetry and perform aggregation with previous or default values").atMost(deduplicationInterval * 2, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -390,7 +394,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset.getId(), device3.getId(), "Contains"); - await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -408,7 +412,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); - await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -432,7 +436,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll configuration.setRelation(new RelationPathLevel(EntitySearchDirection.FROM, "Has")); saveCalculatedField(cf); - await().alias("update relation path and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update relation path and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -458,7 +462,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll configuration.setArguments(Map.of("oc", argument)); saveCalculatedField(cf); - await().alias("update arguments and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update arguments and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -475,7 +479,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device2.getId(), "{\"temperature\":19.6}"); CalculatedField cf = createAvgTemperatureCF(asset.getId()); - await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); @@ -489,7 +493,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll configuration.setMetrics(Map.of("maxTemperature", aggMetric)); saveCalculatedField(cf); - await().alias("update metrics and perform aggregation").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + await().alias("update metrics and perform aggregation").atMost(deduplicationInterval / 2, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("maxTemperature", "24")); @@ -498,7 +502,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device1.getId(), "{\"temperature\":101.3}"); postTelemetry(device2.getId(), "{\"temperature\":25.8}"); - await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("maxTemperature", "26")); @@ -511,7 +515,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device2.getId(), "{\"temperature\":19.6}"); CalculatedField cf = createAvgTemperatureCF(asset.getId()); - await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); @@ -524,7 +528,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll configuration.setOutput(output); saveCalculatedField(cf); - await().alias("update output and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update output and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ArrayNode avgTemperature = getServerAttributes(asset.getId(), "avgTemperature"); @@ -540,17 +544,17 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device2.getId(), "{\"temperature\":19.6}"); CalculatedField cf = createAvgTemperatureCF(asset.getId()); - await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); }); var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); - configuration.setDeduplicationIntervalMillis(2 * deduplicationInterval); + configuration.setDeduplicationIntervalInSec(2 * deduplicationInterval); saveCalculatedField(cf); - await().alias("update deduplication interval and perform aggregation").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + await().alias("update deduplication interval and perform aggregation").atMost(deduplicationInterval / 2, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); @@ -558,7 +562,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device2.getId(), "{\"temperature\":32.1}"); - await().alias("update telemetry and perform aggregation").atMost(2 * deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(2 * deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("avgTemperature", "28")); @@ -566,7 +570,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll } private void checkInitialCalculation() { - await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(this::checkInitialCalculationValues); } @@ -656,7 +660,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll LatestValuesAggregationCalculatedFieldConfiguration configuration = new LatestValuesAggregationCalculatedFieldConfiguration(); configuration.setRelation(relation); configuration.setArguments(inputs); - configuration.setDeduplicationIntervalMillis(deduplicationInterval); + configuration.setDeduplicationIntervalInSec(deduplicationInterval); configuration.setMetrics(metrics); configuration.setOutput(output); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java index f83a812529..6a475daae3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java @@ -40,5 +40,6 @@ public class SystemParams { long maxDataPointsPerRollingArg; int minAllowedScheduledUpdateIntervalInSecForCF; int maxRelationLevelPerCfArgument; + long minAllowedDeduplicationIntervalInSecForCF; TrendzSettings trendzSettings; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java index 721930e676..5f8fb65265 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -29,7 +29,7 @@ public class LatestValuesAggregationCalculatedFieldConfiguration implements Argu private RelationPathLevel relation; private Map arguments; - private long deduplicationIntervalMillis; + private long deduplicationIntervalInSec; private Map metrics; private Output output; private boolean useLatestTs; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index c6bd9a7f38..9f124fd9e4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -184,6 +184,8 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private long maxStateSizeInKBytes = 32; @Schema(example = "2") private long maxSingleValueArgumentSizeInKBytes = 2; + @Schema(example = "3600") + private long minAllowedDeduplicationIntervalInSecForCF = 3600; @Override public long getProfileThreshold(ApiUsageRecordKey key) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index 67ed8f29b0..319a4fbe8b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.configuration.ArgumentsBasedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScheduledUpdateSupportedCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.cf.CalculatedFieldDao; @@ -46,6 +47,7 @@ public class CalculatedFieldDataValidator extends DataValidator validateCalculatedFieldConfiguration(calculatedField); validateSchedulingConfiguration(tenantId, calculatedField); validateRelationQuerySourceArguments(tenantId, calculatedField); + validateAggregationConfiguration(tenantId, calculatedField); } @Override @@ -87,7 +89,7 @@ public class CalculatedFieldDataValidator extends DataValidator private void validateSchedulingConfiguration(TenantId tenantId, CalculatedField calculatedField) { if (!(calculatedField.getConfiguration() instanceof ScheduledUpdateSupportedCalculatedFieldConfiguration scheduledUpdateCfg) - || !scheduledUpdateCfg.isScheduledUpdateEnabled()) { + || !scheduledUpdateCfg.isScheduledUpdateEnabled()) { return; } long minAllowedScheduledUpdateInterval = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMinAllowedScheduledUpdateIntervalInSecForCF); @@ -110,6 +112,17 @@ public class CalculatedFieldDataValidator extends DataValidator wrapAsDataValidation(() -> relationQueryDynamicSourceConfiguration.validateMaxRelationLevel(argumentName, maxRelationLevel))); } + private void validateAggregationConfiguration(TenantId tenantId, CalculatedField calculatedField) { + if (!(calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfiguration)) { + return; + } + long minAllowedDeduplicationInterval = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMinAllowedDeduplicationIntervalInSecForCF); + if (aggConfiguration.getDeduplicationIntervalInSec() < minAllowedDeduplicationInterval) { + throw new IllegalArgumentException("Deduplication interval is less than configured " + + "minimum allowed interval in tenant profile: " + minAllowedDeduplicationInterval); + } + } + private static void wrapAsDataValidation(Runnable validation) { try { validation.run(); From 55fb64ef13af8c7cd84033ce31639087f4cf8ac7 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 20 Oct 2025 11:20:38 +0300 Subject: [PATCH 11/33] fixed delete attributes --- ...CalculatedFieldEntityMessageProcessor.java | 49 +++++++----- ...alculatedFieldManagerMessageProcessor.java | 30 ++++---- ...tValuesAggregationCalculatedFieldTest.java | 75 +++++++++++++++++++ 3 files changed, 119 insertions(+), 35 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 58113deb08..e0faf46185 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -551,19 +551,19 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { - return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getRelatedEntityArguments(), data); + return mapToArguments(entityId, ctx.getMainEntityArguments(), Collections.emptyMap(), data); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { return mapToArguments(entityId, ctx.getLinkedAndDynamicArgs(entityId), ctx.getRelatedEntityArguments(), data); } - private Map mapToArguments(EntityId originator, Map argNames, Map aggArgNames, List data) { + private Map mapToArguments(EntityId originator, Map argNames, Map relatedEntityArgs, List data) { Map arguments = new HashMap<>(); - if (!aggArgNames.isEmpty()) { + if (!relatedEntityArgs.isEmpty()) { for (TsKvProto item : data) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); - String argName = aggArgNames.get(key); + String argName = relatedEntityArgs.get(key); if (argName != null) { arguments.put(argName, new AggSingleEntityArgumentEntry(originator, item)); } @@ -587,17 +587,17 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List attrDataList) { - return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), ctx.getRelatedEntityArguments(), scope, attrDataList); + return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), Collections.emptyMap(), scope, attrDataList); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { var argNames = ctx.getLinkedAndDynamicArgs(entityId); List geofencingArgumentNames = ctx.getLinkedEntityAndCurrentOwnerGeofencingArgumentNames(); - Map aggregationInputs = ctx.getRelatedEntityArguments(); - return mapToArguments(entityId, argNames, geofencingArgumentNames, aggregationInputs, scope, attrDataList); + Map relatedEntityArgs = ctx.getRelatedEntityArguments(); + return mapToArguments(entityId, argNames, geofencingArgumentNames, relatedEntityArgs, scope, attrDataList); } - private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, Map aggArgNames, AttributeScopeProto scope, List attrDataList) { + private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, Map relatedEntityArgs, AttributeScopeProto scope, List attrDataList) { Map arguments = new HashMap<>(); if (!argNames.isEmpty()) { for (AttributeValueProto item : attrDataList) { @@ -613,10 +613,10 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM arguments.put(argName, new SingleValueArgumentEntry(item)); } } - if (!aggArgNames.isEmpty()) { + if (!relatedEntityArgs.isEmpty()) { for (AttributeValueProto item : attrDataList) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); - String argName = aggArgNames.get(key); + String argName = relatedEntityArgs.get(key); if (argName != null) { arguments.put(argName, new AggSingleEntityArgumentEntry(entityId, item)); } @@ -627,26 +627,40 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private Map mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List removedAttrKeys) { var argNames = ctx.getLinkedAndDynamicArgs(entityId); - if (argNames.isEmpty()) { + Map relatedEntityArguments = ctx.getRelatedEntityArguments(); + if (argNames.isEmpty() && relatedEntityArguments.isEmpty()) { return Collections.emptyMap(); } List geofencingArgumentNames = ctx.getLinkedEntityAndCurrentOwnerGeofencingArgumentNames(); - List relatedArgumentNames = ctx.getRelatedEntityArgumentNames(); - return mapToArgumentsWithDefaultValue(entityId, argNames, ctx.getArguments(), geofencingArgumentNames, relatedArgumentNames, scope, removedAttrKeys); + return mapToArgumentsWithDefaultValue(entityId, argNames, ctx.getArguments(), geofencingArgumentNames, relatedEntityArguments, scope, removedAttrKeys); } private Map mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, AttributeScopeProto scope, List removedAttrKeys) { - return mapToArgumentsWithDefaultValue(null, ctx.getMainEntityArguments(), ctx.getArguments(), ctx.getMainEntityGeofencingArgumentNames(), new ArrayList<>(), scope, removedAttrKeys); + return mapToArgumentsWithDefaultValue(null, ctx.getMainEntityArguments(), ctx.getArguments(), ctx.getMainEntityGeofencingArgumentNames(), Collections.emptyMap(), scope, removedAttrKeys); } private Map mapToArgumentsWithDefaultValue(EntityId msgEntityId, Map argNames, Map configArguments, List geofencingArgNames, - List relatedEntityArgNames, + Map relatedEntityArgs, AttributeScopeProto scope, List removedAttrKeys) { Map arguments = new HashMap<>(); + if (!relatedEntityArgs.isEmpty()) { + for (String removedKey : removedAttrKeys) { + ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + if (relatedEntityArgs.containsKey(key)) { + String argName = relatedEntityArgs.get(key); + Argument argument = configArguments.get(argName); + String defaultValue = (argument != null) ? argument.getDefaultValue() : null; + SingleValueArgumentEntry argumentEntry = StringUtils.isNotEmpty(defaultValue) + ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) + : new SingleValueArgumentEntry(); + arguments.put(argName, new AggSingleEntityArgumentEntry(msgEntityId, argumentEntry)); + } + } + } for (String removedKey : removedAttrKeys) { ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); String argName = argNames.get(key); @@ -662,12 +676,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM SingleValueArgumentEntry argumentEntry = StringUtils.isNotEmpty(defaultValue) ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) : new SingleValueArgumentEntry(); - if (relatedEntityArgNames.contains(argName)) { - arguments.put(argName, new AggSingleEntityArgumentEntry(msgEntityId, argumentEntry)); - continue; - } arguments.put(argName, argumentEntry); - } return arguments; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index af5a672157..24fd5d320b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -41,9 +41,9 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationPathLevel; -import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.msg.CalculatedFieldStatePartitionRestoreMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; @@ -520,22 +520,22 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware List result = new ArrayList<>(); if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration configuration) { RelationPathLevel relation = configuration.getRelation(); - switch (relation.direction()) { - case FROM -> { - List byToAndType = relationService.findByToAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); - if (byToAndType != null && !byToAndType.isEmpty()) { - EntityRelation entityRelation = byToAndType.get(0); // only one supported + EntitySearchDirection inverseDirection = switch (relation.direction()) { + case FROM -> EntitySearchDirection.TO; + case TO -> EntitySearchDirection.FROM; + }; + RelationPathLevel inverseRelation = new RelationPathLevel(inverseDirection, relation.relationType()); + List byRelationPathQuery = relationService.findByRelationPathQuery(tenantId, new EntityRelationPathQuery(entityId, List.of(inverseRelation))); + if (byRelationPathQuery != null && !byRelationPathQuery.isEmpty()) { + switch (relation.direction()) { + case FROM -> { + EntityRelation entityRelation = byRelationPathQuery.get(0); // only one supported result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getFrom())); } - } - case TO -> { - List byFromAndType = relationService.findByFromAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); - if (byFromAndType != null && !byFromAndType.isEmpty()) { - for (EntityRelation entityRelation : byFromAndType) { - if (entityRelation.getTo().equals(cf.getEntityId())) { - result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getTo())); - } - } + case TO -> { + byRelationPathQuery.stream() + .filter(entityRelation -> entityRelation.getTo().equals(cf.getEntityId())) + .forEach(entityRelation -> result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getTo()))); } } } diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java index e8c244b57e..eb11a329be 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -383,6 +383,44 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll }); } + @Test + public void testDeleteAttr_checkAggregationWithDefault() throws Exception { + Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + Device device3 = createDevice("Device 3", "1234567890333"); + Device device4 = createDevice("Device 4", "1234567890444"); + + createEntityRelation(asset2.getId(), device3.getId(), "Contains"); + createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + + postAttributes(device3.getId(), AttributeScope.SERVER_SCOPE, "{\"occupied\":true}"); + postAttributes(device4.getId(), AttributeScope.SERVER_SCOPE, "{\"occupied\":true}"); + + createOccupancyCFWithAttr(asset2.getId()); + + await().alias("create CF and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "0", + "occupiedSpaces", "2", + "totalSpaces", "2" + )); + }); + + doDelete("/api/plugins/telemetry/DEVICE/" + device3.getUuidId() + "/SERVER_SCOPE?keys=occupied", String.class); + doDelete("/api/plugins/telemetry/DEVICE/" + device4.getUuidId() + "/SERVER_SCOPE?keys=occupied", String.class); + + await().alias("delete attribute and perform aggregation with default values").atMost(deduplicationInterval * 2, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); + }); + } + @Test public void testCreateRelation_checkAggregation() throws Exception { createOccupancyCF(asset.getId()); @@ -646,6 +684,43 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll output); } + private CalculatedField createOccupancyCFWithAttr(EntityId entityId) { + Map arguments = new HashMap<>(); + Argument argument = new Argument(); + argument.setRefEntityKey(new ReferencedEntityKey("occupied", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + argument.setDefaultValue("false"); + arguments.put("oc", argument); + + Map aggMetrics = new HashMap<>(); + + AggMetric freeSpaces = new AggMetric(); + freeSpaces.setFunction(AggFunction.COUNT); + freeSpaces.setFilter("return oc == false;"); + freeSpaces.setInput(new AggKeyInput("oc")); + aggMetrics.put("freeSpaces", freeSpaces); + + AggMetric occupiedSpaces = new AggMetric(); + occupiedSpaces.setFunction(AggFunction.COUNT); + occupiedSpaces.setFilter("return oc == true;"); + occupiedSpaces.setInput(new AggKeyInput("oc")); + aggMetrics.put("occupiedSpaces", occupiedSpaces); + + AggMetric totalSpaces = new AggMetric(); + totalSpaces.setFunction(AggFunction.COUNT); + totalSpaces.setInput(new AggFunctionInput("return 1;")); + aggMetrics.put("totalSpaces", totalSpaces); + + Output output = new Output(); + output.setType(OutputType.TIME_SERIES); + output.setDecimalsByDefault(0); + + return createAggCf("Occupied spaces", entityId, + new RelationPathLevel(EntitySearchDirection.FROM, "Contains"), + arguments, + aggMetrics, + output); + } + private CalculatedField createAggCf(String name, EntityId entityId, RelationPathLevel relation, From 5e8de6b955f432a244dbb5c4406bc970ee18c5a8 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Oct 2025 10:33:20 +0300 Subject: [PATCH 12/33] renamed cf --- ...CalculatedFieldEntityMessageProcessor.java | 12 +++++----- ...alculatedFieldManagerMessageProcessor.java | 12 +++++----- ...tractCalculatedFieldProcessingService.java | 8 +++---- .../cf/DefaultCalculatedFieldCache.java | 6 ++--- .../DefaultCalculatedFieldQueueService.java | 4 ++-- .../cf/ctx/state/CalculatedFieldCtx.java | 22 +++++++++---------- ...itiesAggregationCalculatedFieldState.java} | 10 ++++----- .../utils/CalculatedFieldArgumentUtils.java | 4 ++-- .../server/utils/CalculatedFieldUtils.java | 10 ++++----- ...titiesAggregationCalculatedFieldTest.java} | 18 +++++++-------- .../common/data/cf/CalculatedFieldType.java | 2 +- .../CalculatedFieldConfiguration.java | 4 ++-- ...regationCalculatedFieldConfiguration.java} | 4 ++-- .../CalculatedFieldDataValidator.java | 4 ++-- 14 files changed, 60 insertions(+), 60 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/{LatestValuesAggregationCalculatedFieldState.java => RelaredEntitiesAggregationCalculatedFieldState.java} (94%) rename application/src/test/java/org/thingsboard/server/cf/{LatestValuesAggregationCalculatedFieldTest.java => RelatedEntitiesAggregationCalculatedFieldTest.java} (97%) rename common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/{LatestValuesAggregationCalculatedFieldConfiguration.java => RelatedEntitiesAggregationCalculatedFieldConfiguration.java} (91%) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 1252b8091b..5a1ada08c5 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -54,7 +54,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelaredEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -254,8 +254,8 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM Map fetchedArgs = fetchAggArguments(ctx, relatedEntityId); Map updatedArgs = state.update(fetchedArgs, ctx); - if (state instanceof LatestValuesAggregationCalculatedFieldState latestValuesState) { - latestValuesState.setLastMetricsEvalTs(-1); + if (state instanceof RelaredEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { + relatedEntitiesAggState.setLastMetricsEvalTs(-1); } state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); @@ -271,7 +271,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM msg.getCallback().onSuccess(); return; } - if (state instanceof LatestValuesAggregationCalculatedFieldState aggState) { + if (state instanceof RelaredEntitiesAggregationCalculatedFieldState aggState) { cleanupAggregationState(msg.getRelatedEntityId(), aggState); processStateIfReady(state, Collections.emptyMap(), state.getCtx(), Collections.emptyList(), null, null, msg.getCallback()); } else { @@ -279,7 +279,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } - private void cleanupAggregationState(EntityId relatedEntityId, LatestValuesAggregationCalculatedFieldState state) { + private void cleanupAggregationState(EntityId relatedEntityId, RelaredEntitiesAggregationCalculatedFieldState state) { state.getArguments().values().forEach(argEntry -> { AggArgumentEntry aggEntry = (AggArgumentEntry) argEntry; aggEntry.getAggInputs().remove(relatedEntityId); @@ -691,7 +691,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); - if (CalculatedFieldType.LATEST_VALUES_AGGREGATION.equals(ctx.getCfType())) { + if (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION.equals(ctx.getCfType())) { fetchedArgs = fetchedArgs.entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 24fd5d320b..5b4528370b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; @@ -346,7 +346,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware List matchingCfs = cfsByEntityIdAndProfile.stream() .filter(cf -> { - var config = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + var config = (RelatedEntitiesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); RelationPathLevel relation = config.getRelation(); return direction.equals(relation.direction()) && relationType.equals(relation.relationType()); }) @@ -377,7 +377,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } calculatedFields.put(cf.getId(), cfCtx); - if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + if (cf.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig) { aggCalculatedFields.put(cf.getId(), cfCtx); } // We use copy on write lists to safely pass the reference to another actor for the iteration. @@ -411,7 +411,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(newCfCtx).eventEntity(newCfCtx.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(newCf.getId(), newCfCtx); - if (newCf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + if (newCf.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration) { aggCalculatedFields.put(newCf.getId(), newCfCtx); } List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); @@ -518,7 +518,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private List findRelationsForCf(EntityId entityId, CalculatedFieldCtx cf) { List result = new ArrayList<>(); - if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration configuration) { + if (cf.getCalculatedField().getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration configuration) { RelationPathLevel relation = configuration.getRelation(); EntitySearchDirection inverseDirection = switch (relation.direction()) { case FROM -> EntitySearchDirection.TO; @@ -753,7 +753,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(cf.getId(), cfCtx); - if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + if (cf.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration) { aggCalculatedFields.put(cf.getId(), cfCtx); } // We use copy on write lists to safely pass the reference to another actor for the iteration. diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 1c606cf840..3ddc8e1067 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -27,7 +27,7 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; @@ -98,7 +98,7 @@ public abstract class AbstractCalculatedFieldProcessingService { Map> argFutures = switch (ctx.getCfType()) { case GEOFENCING -> fetchGeofencingCalculatedFieldArguments(ctx, entityId, false, ts); case SIMPLE, SCRIPT, ALARM, PROPAGATION -> getBaseCalculatedFieldArguments(ctx, entityId, ts); - case LATEST_VALUES_AGGREGATION -> fetchAggArguments(ctx, entityId, ts); + case RELATED_ENTITIES_AGGREGATION -> fetchAggArguments(ctx, entityId, ts); }; if (ctx.getCfType() == PROPAGATION) { argFutures.put(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId)); @@ -190,7 +190,7 @@ public abstract class AbstractCalculatedFieldProcessingService { } protected Map> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { - LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig = (RelatedEntitiesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); ListenableFuture> relatedEntitiesFut = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getRelation()); @@ -202,7 +202,7 @@ public abstract class AbstractCalculatedFieldProcessingService { } protected ListenableFuture> fetchEntityAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { - LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig = (RelatedEntitiesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); Map> argsFutures = aggConfig.getArguments().entrySet().stream() .collect(Collectors.toMap( diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 918d71e326..1c755ee05a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; @@ -83,7 +83,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { cfs.forEach(cf -> { if (cf != null) { calculatedFields.putIfAbsent(cf.getId(), cf); - if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + if (cf.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration) { aggCalculatedFields.put(cf.getId(), cf); } } @@ -200,7 +200,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); - if (configuration instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + if (configuration instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration) { aggCalculatedFields.put(calculatedField.getId(), calculatedField); } calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 147882c3c2..84f49c7c9e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -27,7 +27,7 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -190,7 +190,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS List cfCtxs = calculatedFieldCache.getAggCalculatedFieldCtxsByFilter(relatedEntityFilter); for (CalculatedFieldCtx cfCtx : cfCtxs) { - if (cfCtx.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + if (cfCtx.getCalculatedField().getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig) { RelationPathLevel relation = aggConfig.getRelation(); EntitySearchDirection inverseDirection = switch (relation.direction()) { case FROM -> EntitySearchDirection.TO; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 89ed4e4401..46442847a0 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -44,7 +44,7 @@ import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.ScheduledUpdateSupportedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -139,7 +139,7 @@ public class CalculatedFieldCtx { var refId = entry.getValue().getRefEntityId(); var refKey = entry.getValue().getRefEntityKey(); if (refId == null) { - if (CalculatedFieldType.LATEST_VALUES_AGGREGATION.equals(cfType)) { + if (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION.equals(cfType)) { relatedEntityArguments.put(refKey, entry.getKey()); continue; } @@ -185,7 +185,7 @@ public class CalculatedFieldCtx { this.scheduledUpdateIntervalMillis = scheduledConfig.isScheduledUpdateEnabled() ? TimeUnit.SECONDS.toMillis(scheduledConfig.getScheduledUpdateInterval()) : -1L; } this.requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); - if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + if (calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig) { this.useLatestTs = aggConfig.isUseLatestTs(); } this.systemContext = systemContext; @@ -228,8 +228,8 @@ public class CalculatedFieldCtx { } initialized = true; } - case LATEST_VALUES_AGGREGATION -> { - LatestValuesAggregationCalculatedFieldConfiguration configuration = (LatestValuesAggregationCalculatedFieldConfiguration) calculatedField.getConfiguration(); + case RELATED_ENTITIES_AGGREGATION -> { + RelatedEntitiesAggregationCalculatedFieldConfiguration configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) calculatedField.getConfiguration(); configuration.getMetrics().forEach((key, metric) -> { if (metric.getInput() instanceof AggFunctionInput functionInput) { initTbelExpression(functionInput.getFunction()); @@ -594,8 +594,8 @@ public class CalculatedFieldCtx { if (scheduledUpdateIntervalMillis != other.scheduledUpdateIntervalMillis) { return true; } - if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig - && other.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig + if (calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration thisConfig + && other.getCalculatedField().getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration otherConfig && (thisConfig.getDeduplicationIntervalInSec() != otherConfig.getDeduplicationIntervalInSec() || !thisConfig.getMetrics().equals(otherConfig.getMetrics()))) { return true; } @@ -617,7 +617,7 @@ public class CalculatedFieldCtx { if (hasGeofencingZoneGroupConfigurationChanges(other)) { return true; } - if (hasLatestValuesAggregationConfigurationChanges(other)) { + if (hasRelatedEntitiesAggregationConfigurationChanges(other)) { return true; } return false; @@ -631,9 +631,9 @@ public class CalculatedFieldCtx { return false; } - private boolean hasLatestValuesAggregationConfigurationChanges(CalculatedFieldCtx other) { - if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig - && other.calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig) { + private boolean hasRelatedEntitiesAggregationConfigurationChanges(CalculatedFieldCtx other) { + if (calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration thisConfig + && other.calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration otherConfig) { return !thisConfig.getArguments().equals(otherConfig.getArguments()) || !thisConfig.getRelation().equals(otherConfig.getRelation()); } return false; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelaredEntitiesAggregationCalculatedFieldState.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelaredEntitiesAggregationCalculatedFieldState.java index fdc045a0db..b53042b1fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelaredEntitiesAggregationCalculatedFieldState.java @@ -30,7 +30,7 @@ import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFuncti import org.thingsboard.server.common.data.cf.configuration.aggregation.AggInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; @@ -46,7 +46,7 @@ import java.util.Map.Entry; @Slf4j @Getter -public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedFieldState { +public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculatedFieldState { @Setter private long lastArgsRefreshTs = -1; @@ -57,14 +57,14 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF private final Map> inputs = new HashMap<>(); - public LatestValuesAggregationCalculatedFieldState(EntityId entityId) { + public RelaredEntitiesAggregationCalculatedFieldState(EntityId entityId) { super(entityId); } @Override public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) { super.setCtx(ctx, actorCtx); - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); metrics = configuration.getMetrics(); deduplicationInterval = configuration.getDeduplicationIntervalInSec(); } @@ -86,7 +86,7 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF @Override public CalculatedFieldType getType() { - return CalculatedFieldType.LATEST_VALUES_AGGREGATION; + return CalculatedFieldType.RELATED_ENTITIES_AGGREGATION; } @Override diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index d30b4eaf93..e1763ffa2d 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -35,7 +35,7 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelaredEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalculatedFieldState; @@ -91,7 +91,7 @@ public class CalculatedFieldArgumentUtils { case GEOFENCING -> new GeofencingCalculatedFieldState(entityId); case ALARM -> new AlarmCalculatedFieldState(entityId); case PROPAGATION -> new PropagationCalculatedFieldState(entityId); - case LATEST_VALUES_AGGREGATION -> new LatestValuesAggregationCalculatedFieldState(entityId); + case RELATED_ENTITIES_AGGREGATION -> new RelaredEntitiesAggregationCalculatedFieldState(entityId); }; } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 6e65b4d5d9..67af13fdb3 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -49,7 +49,7 @@ import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelaredEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; @@ -120,7 +120,7 @@ public class CalculatedFieldUtils { alarmStateProto.setClearRuleState(toAlarmRuleStateProto(alarmState.getClearRuleState())); } } - if (state instanceof LatestValuesAggregationCalculatedFieldState aggState) { + if (state instanceof RelaredEntitiesAggregationCalculatedFieldState aggState) { aggBuilder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); builder.setLatestValuesAggregationState(aggBuilder.build()); } @@ -214,7 +214,7 @@ public class CalculatedFieldUtils { case GEOFENCING -> new GeofencingCalculatedFieldState(id.entityId()); case ALARM -> new AlarmCalculatedFieldState(id.entityId()); case PROPAGATION -> new PropagationCalculatedFieldState(id.entityId()); - case LATEST_VALUES_AGGREGATION -> new LatestValuesAggregationCalculatedFieldState(id.entityId()); + case RELATED_ENTITIES_AGGREGATION -> new RelaredEntitiesAggregationCalculatedFieldState(id.entityId()); }; proto.getSingleValueArgumentsList().forEach(argProto -> @@ -240,8 +240,8 @@ public class CalculatedFieldUtils { alarmState.setClearRuleState(fromAlarmRuleStateProto(alarmStateProto.getClearRuleState(), alarmState)); } } - case LATEST_VALUES_AGGREGATION -> { - LatestValuesAggregationCalculatedFieldState aggState = (LatestValuesAggregationCalculatedFieldState) state; + case RELATED_ENTITIES_AGGREGATION -> { + RelaredEntitiesAggregationCalculatedFieldState aggState = (RelaredEntitiesAggregationCalculatedFieldState) state; LatestValuesAggregationStateProto aggregationStateProto = proto.getLatestValuesAggregationState(); Map> arguments = new HashMap<>(); aggregationStateProto.getAggArgumentsList().forEach(argProto -> { diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/RelatedEntitiesAggregationCalculatedFieldTest.java similarity index 97% rename from application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java rename to application/src/test/java/org/thingsboard/server/cf/RelatedEntitiesAggregationCalculatedFieldTest.java index eb11a329be..068c5f851d 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/RelatedEntitiesAggregationCalculatedFieldTest.java @@ -39,7 +39,7 @@ import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFuncti import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; @@ -66,7 +66,7 @@ import static org.thingsboard.server.cf.CalculatedFieldIntegrationTest.POLL_INTE @Slf4j @DaoSqlTest -public class LatestValuesAggregationCalculatedFieldTest extends AbstractControllerTest { +public class RelatedEntitiesAggregationCalculatedFieldTest extends AbstractControllerTest { private Tenant savedTenant; @@ -470,7 +470,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset.getId(), device3.getId(), "Has"); postTelemetry(device3.getId(), "{\"occupied\":true}"); - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); configuration.setRelation(new RelationPathLevel(EntitySearchDirection.FROM, "Has")); saveCalculatedField(cf); @@ -493,7 +493,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device1.getId(), "{\"occupiedStatus\":false}"); postTelemetry(device2.getId(), "{\"occupiedStatus\":false}"); - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); Argument argument = new Argument(); argument.setRefEntityKey(new ReferencedEntityKey("oc", ArgumentType.TS_LATEST, null)); argument.setDefaultValue("false"); @@ -523,7 +523,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); }); - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); AggMetric aggMetric = new AggMetric(); aggMetric.setInput(new AggKeyInput("temp")); aggMetric.setFilter("return temp < 100;"); @@ -559,7 +559,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); }); - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); Output output = new Output(); output.setType(OutputType.ATTRIBUTES); output.setScope(AttributeScope.SERVER_SCOPE); @@ -588,7 +588,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); }); - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); configuration.setDeduplicationIntervalInSec(2 * deduplicationInterval); saveCalculatedField(cf); @@ -730,9 +730,9 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll CalculatedField calculatedField = new CalculatedField(); calculatedField.setName(name); calculatedField.setEntityId(entityId); - calculatedField.setType(CalculatedFieldType.LATEST_VALUES_AGGREGATION); + calculatedField.setType(CalculatedFieldType.RELATED_ENTITIES_AGGREGATION); - LatestValuesAggregationCalculatedFieldConfiguration configuration = new LatestValuesAggregationCalculatedFieldConfiguration(); + RelatedEntitiesAggregationCalculatedFieldConfiguration configuration = new RelatedEntitiesAggregationCalculatedFieldConfiguration(); configuration.setRelation(relation); configuration.setArguments(inputs); configuration.setDeduplicationIntervalInSec(deduplicationInterval); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index fe9ee7f9aa..4463c835db 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -26,7 +26,7 @@ public enum CalculatedFieldType { GEOFENCING, ALARM, PROPAGATION, - LATEST_VALUES_AGGREGATION; + RELATED_ENTITIES_AGGREGATION; public static final Set all = Collections.unmodifiableSet(EnumSet.allOf(CalculatedFieldType.class)); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 3072b7c546..ca0a3c1a54 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -42,7 +42,7 @@ import java.util.stream.Collectors; @Type(value = GeofencingCalculatedFieldConfiguration.class, name = "GEOFENCING"), @Type(value = AlarmCalculatedFieldConfiguration.class, name = "ALARM"), @Type(value = PropagationCalculatedFieldConfiguration.class, name = "PROPAGATION"), - @Type(value = LatestValuesAggregationCalculatedFieldConfiguration.class, name = "LATEST_VALUES_AGGREGATION") + @Type(value = RelatedEntitiesAggregationCalculatedFieldConfiguration.class, name = "RELATED_ENTITIES_AGGREGATION") }) @JsonIgnoreProperties(ignoreUnknown = true) public interface CalculatedFieldConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java similarity index 91% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java index 5f8fb65265..69e4ee7fdf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.relation.RelationPathLevel; import java.util.Map; @Data -public class LatestValuesAggregationCalculatedFieldConfiguration implements ArgumentsBasedCalculatedFieldConfiguration { +public class RelatedEntitiesAggregationCalculatedFieldConfiguration implements ArgumentsBasedCalculatedFieldConfiguration { private RelationPathLevel relation; private Map arguments; @@ -36,7 +36,7 @@ public class LatestValuesAggregationCalculatedFieldConfiguration implements Argu @Override public CalculatedFieldType getType() { - return CalculatedFieldType.LATEST_VALUES_AGGREGATION; + return CalculatedFieldType.RELATED_ENTITIES_AGGREGATION; } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index 319a4fbe8b..3ccb837b3f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -21,7 +21,7 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.configuration.ArgumentsBasedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScheduledUpdateSupportedCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.cf.CalculatedFieldDao; @@ -113,7 +113,7 @@ public class CalculatedFieldDataValidator extends DataValidator } private void validateAggregationConfiguration(TenantId tenantId, CalculatedField calculatedField) { - if (!(calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfiguration)) { + if (!(calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfiguration)) { return; } long minAllowedDeduplicationInterval = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMinAllowedDeduplicationIntervalInSecForCF); From 2778f79e5e8a1d9262450a33edcfbc77d64e88c2 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Oct 2025 12:45:29 +0300 Subject: [PATCH 13/33] minor refactoring --- ...CalculatedFieldEntityMessageProcessor.java | 80 +++++++------------ ...alculatedFieldManagerMessageProcessor.java | 4 +- ...tractCalculatedFieldProcessingService.java | 14 ---- .../cf/CalculatedFieldProcessingService.java | 2 - ...faultCalculatedFieldProcessingService.java | 5 -- .../service/cf/ctx/state/ArgumentEntry.java | 6 +- .../cf/ctx/state/ArgumentEntryType.java | 2 +- .../AggSingleEntityArgumentEntry.java | 1 - ...itiesAggregationCalculatedFieldState.java} | 64 ++++++++------- ...java => RelatedEntitiesArgumentEntry.java} | 20 ++--- .../utils/CalculatedFieldArgumentUtils.java | 4 +- .../server/utils/CalculatedFieldUtils.java | 18 ++--- ... => RelatedEntitiesArgumentEntryTest.java} | 27 ++----- 13 files changed, 96 insertions(+), 151 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/{RelaredEntitiesAggregationCalculatedFieldState.java => RelatedEntitiesAggregationCalculatedFieldState.java} (84%) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/{AggArgumentEntry.java => RelatedEntitiesArgumentEntry.java} (72%) rename application/src/test/java/org/thingsboard/server/service/cf/ctx/state/{AggArgumentEntryTest.java => RelatedEntitiesArgumentEntryTest.java} (80%) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 5a1ada08c5..434f555325 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -52,9 +52,8 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.RelaredEntitiesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -227,17 +226,19 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); var state = states.get(ctx.getCfId()); try { - boolean justRestored = false; + Map updatedArgs = new HashMap<>(); if (state == null) { state = createState(ctx); - justRestored = true; + } else { + if (state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { + Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, msg.getRelatedEntityId(), ctx.getArguments()); + updatedArgs = relatedEntitiesAggState.updateEntityData(toAggSingleEntityArguments(msg.getRelatedEntityId(), fetchedArgs)); + } + + state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); } if (state.isSizeOk()) { - Map updatedArgs = new HashMap<>(); - if (!justRestored) { - updatedArgs = updateAggregationState(msg.getRelatedEntityId(), state, ctx); - } - processStateIfReady(state, updatedArgs, ctx, new ArrayList<>(), null, null, callback); + processStateIfReady(state, updatedArgs, ctx, Collections.singletonList(ctx.getCfId()), null, null, callback); } else { throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(); } @@ -250,19 +251,6 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } - private Map updateAggregationState(EntityId relatedEntityId, CalculatedFieldState state, CalculatedFieldCtx ctx) { - Map fetchedArgs = fetchAggArguments(ctx, relatedEntityId); - Map updatedArgs = state.update(fetchedArgs, ctx); - - if (state instanceof RelaredEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { - relatedEntitiesAggState.setLastMetricsEvalTs(-1); - } - - state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); - - return updatedArgs; - } - private void handleRelationDelete(CalculatedFieldRelatedEntityMsg msg) throws CalculatedFieldException { CalculatedFieldCtx ctx = msg.getCalculatedField(); CalculatedFieldId cfId = ctx.getCfId(); @@ -271,34 +259,22 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM msg.getCallback().onSuccess(); return; } - if (state instanceof RelaredEntitiesAggregationCalculatedFieldState aggState) { - cleanupAggregationState(msg.getRelatedEntityId(), aggState); - processStateIfReady(state, Collections.emptyMap(), state.getCtx(), Collections.emptyList(), null, null, msg.getCallback()); + if (state instanceof RelatedEntitiesAggregationCalculatedFieldState aggState) { + aggState.cleanupEntityData(msg.getRelatedEntityId()); + + state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); + + if (state.isSizeOk()) { + processStateIfReady(state, Collections.emptyMap(), ctx, Collections.singletonList(ctx.getCfId()), null, null, msg.getCallback()); + } else { + throw new RuntimeException(ctx.getSizeExceedsLimitMessage()); + } } else { + // todo: log msg.getCallback().onSuccess(); } } - private void cleanupAggregationState(EntityId relatedEntityId, RelaredEntitiesAggregationCalculatedFieldState state) { - state.getArguments().values().forEach(argEntry -> { - AggArgumentEntry aggEntry = (AggArgumentEntry) argEntry; - aggEntry.getAggInputs().remove(relatedEntityId); - }); - state.getInputs().remove(relatedEntityId); - state.setLastMetricsEvalTs(-1); - } - - @SneakyThrows - private Map fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { - ListenableFuture> argumentsFuture = cfService.fetchAggEntityArguments(ctx, entityId); - // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. - // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. - // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, - // but this will significantly complicate the code. - return argumentsFuture.get(1, TimeUnit.MINUTES); - } - - public void process(EntityCalculatedFieldTelemetryMsg msg) throws CalculatedFieldException { log.trace("[{}] Processing CF telemetry msg: {}", msg.getEntityId(), msg); var proto = msg.getProto(); @@ -692,17 +668,21 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); if (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION.equals(ctx.getCfType())) { - fetchedArgs = fetchedArgs.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - argEntry -> new AggSingleEntityArgumentEntry(entityId, argEntry.getValue()) - )); + fetchedArgs = toAggSingleEntityArguments(entityId, fetchedArgs); } fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); return fetchedArgs; } + private Map toAggSingleEntityArguments(EntityId relatedEntityId, Map fetchedArgs) { + return fetchedArgs.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + argEntry -> new AggSingleEntityArgumentEntry(relatedEntityId, argEntry.getValue()) + )); + } + private static List getCalculatedFieldIds(CalculatedFieldTelemetryMsgProto proto) { List cfIds = new LinkedList<>(); for (var cfId : proto.getPreviousCalculatedFieldsList()) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 5b4528370b..7f1c7b1925 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -469,8 +469,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); log.debug("Received telemetry msg from entity [{}]", entityId); - // 3 = 1 for CF processing + 1 for links processing + 1 for owner entity processing - MultipleTbCallback callback = new MultipleTbCallback(3, msg.getCallback()); + // 4 = 1 for CF processing + 1 for links processing + 1 for owner entity processing + 1 for aggregation processing + MultipleTbCallback callback = new MultipleTbCallback(4, msg.getCallback()); // process all cfs related to entity, or it's profile; var entityIdFields = getCalculatedFieldsByEntityId(entityId); var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 3ddc8e1067..0c0f2f23e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -201,20 +201,6 @@ public abstract class AbstractCalculatedFieldProcessingService { )); } - protected ListenableFuture> fetchEntityAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { - RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig = (RelatedEntitiesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - - Map> argsFutures = aggConfig.getArguments().entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> fetchSingleAggArgumentEntry(ctx.getTenantId(), entityId, entry.getValue(), ts) - )); - - return Futures.whenAllComplete(argsFutures.values()) - .call(() -> resolveArgumentFutures(argsFutures), - MoreExecutors.directExecutor()); - } - private ListenableFuture> resolveGeofencingEntityIds(TenantId tenantId, EntityId entityId, Map.Entry entry) { Argument value = entry.getValue(); if (value.getRefEntityId() != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index 52b3341151..a9139572b8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -33,8 +33,6 @@ public interface CalculatedFieldProcessingService { ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId); - ListenableFuture> fetchAggEntityArguments(CalculatedFieldCtx ctx, EntityId entityId); - Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId); Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index 9074c1036f..d7957dce9b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -91,11 +91,6 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF return super.fetchArguments(ctx, entityId, System.currentTimeMillis()); } - @Override - public ListenableFuture> fetchAggEntityArguments(CalculatedFieldCtx ctx, EntityId entityId) { - return super.fetchEntityAggArguments(ctx, entityId, System.currentTimeMillis()); - } - @Override public Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId) { return switch (ctx.getCfType()) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index c8f7dd0c3d..5bb16292be 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -22,7 +22,7 @@ import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry; @@ -40,7 +40,7 @@ import java.util.Map; @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING"), @JsonSubTypes.Type(value = PropagationArgumentEntry.class, name = "PROPAGATION"), - @JsonSubTypes.Type(value = AggArgumentEntry.class, name = "AGGREGATE_LATEST"), + @JsonSubTypes.Type(value = RelatedEntitiesArgumentEntry.class, name = "AGGREGATE_LATEST"), @JsonSubTypes.Type(value = AggSingleEntityArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") }) public interface ArgumentEntry { @@ -77,7 +77,7 @@ public interface ArgumentEntry { } static ArgumentEntry createAggArgument(Map entityIdkvEntryMap) { - return new AggArgumentEntry(entityIdkvEntryMap, false); + return new RelatedEntitiesArgumentEntry(entityIdkvEntryMap, false); } static ArgumentEntry createAggSingleArgument(EntityId entityId, KvEntry kvEntry) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java index 9882b8181b..5c672cf04e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java @@ -16,5 +16,5 @@ package org.thingsboard.server.service.cf.ctx.state; public enum ArgumentEntryType { - SINGLE_VALUE, TS_ROLLING, GEOFENCING, PROPAGATION, AGGREGATE_LATEST, AGGREGATE_LATEST_SINGLE + SINGLE_VALUE, TS_ROLLING, GEOFENCING, PROPAGATION, RELATED_ENTITIES, AGGREGATE_LATEST_SINGLE } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java index 6430fe3f1c..b935256860 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java @@ -33,7 +33,6 @@ import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; public class AggSingleEntityArgumentEntry extends SingleValueArgumentEntry { private EntityId entityId; - private boolean deleted; public AggSingleEntityArgumentEntry(EntityId entityId, ArgumentEntry entry) { super(entry); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelaredEntitiesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java similarity index 84% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelaredEntitiesAggregationCalculatedFieldState.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java index b53042b1fe..c0baca836c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelaredEntitiesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -46,7 +45,7 @@ import java.util.Map.Entry; @Slf4j @Getter -public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculatedFieldState { +public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculatedFieldState { @Setter private long lastArgsRefreshTs = -1; @@ -55,9 +54,7 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat private long deduplicationInterval = -1; private Map metrics; - private final Map> inputs = new HashMap<>(); - - public RelaredEntitiesAggregationCalculatedFieldState(EntityId entityId) { + public RelatedEntitiesAggregationCalculatedFieldState(EntityId entityId) { super(entityId); } @@ -75,7 +72,6 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat lastArgsRefreshTs = -1; lastMetricsEvalTs = -1; metrics = null; - inputs.clear(); } @Override @@ -91,17 +87,8 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat @Override public Map update(Map argumentValues, CalculatedFieldCtx ctx) { - Map updatedArguments = super.update(argumentValues, ctx); lastArgsRefreshTs = System.currentTimeMillis(); - for (Map.Entry argEntry : arguments.entrySet()) { - String key = argEntry.getKey(); - AggArgumentEntry aggArgumentEntry = (AggArgumentEntry) argEntry.getValue(); - Map aggInputs = aggArgumentEntry.getAggInputs(); - aggInputs.forEach((entityId, argumentEntry) -> { - inputs.computeIfAbsent(entityId, k -> new HashMap<>()).put(key, argumentEntry); - }); - } - return updatedArguments; + return super.update(argumentValues, ctx); } @Override @@ -115,7 +102,7 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() .type(output.getType()) .scope(output.getScope()) - .result(createResultJson(ctx.isUseLatestTs(), aggResult)) + .result(toSimpleResult(ctx.isUseLatestTs(), aggResult)) .build()); } else { return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() @@ -124,20 +111,47 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat } } + public Map updateEntityData(Map fetchedArgs) { + lastMetricsEvalTs = -1; + return update(fetchedArgs, ctx); + } + + public void cleanupEntityData(EntityId relatedEntityId) { + arguments.values().forEach(argEntry -> { + RelatedEntitiesArgumentEntry aggEntry = (RelatedEntitiesArgumentEntry) argEntry; + aggEntry.getAggInputs().remove(relatedEntityId); + }); + lastMetricsEvalTs = -1; + lastArgsRefreshTs = System.currentTimeMillis(); + } + private boolean shouldRecalculate() { boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - deduplicationInterval; boolean argsUpdatedDuringInterval = lastArgsRefreshTs > lastMetricsEvalTs; return intervalPassed && argsUpdatedDuringInterval; } + private Map> prepareInputs() { + Map> inputs = new HashMap<>(); + for (Map.Entry argEntry : arguments.entrySet()) { + String key = argEntry.getKey(); + RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry.getValue(); + relatedEntitiesArgumentEntry.getAggInputs().forEach((entityId, argumentEntry) -> { + inputs.computeIfAbsent(entityId, k -> new HashMap<>()).put(key, argumentEntry); + }); + } + return inputs; + } + private ObjectNode aggregateMetrics(Output output) throws Exception { ObjectNode aggResult = JacksonUtil.newObjectNode(); + Map> inputs = prepareInputs(); for (Entry entry : metrics.entrySet()) { String metricKey = entry.getKey(); AggMetric metric = entry.getValue(); AggEntry aggMetricEntry = AggFunctionFactory.createAggFunction(metric.getFunction()); - aggregateMetric(metric, aggMetricEntry); + aggregateMetric(metric, aggMetricEntry, inputs); aggMetricEntry.result().ifPresent(result -> { aggResult.set(metricKey, JacksonUtil.valueToTree(formatResult(result, output.getDecimalsByDefault()))); }); @@ -145,7 +159,7 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat return aggResult; } - private void aggregateMetric(AggMetric metric, AggEntry aggEntry) throws Exception { + private void aggregateMetric(AggMetric metric, AggEntry aggEntry, Map> inputs) throws Exception { for (Map entityInputs : inputs.values()) { if (applyAggregation(metric.getFilter(), entityInputs)) { Object arg = resolveAggregationInput(metric.getInput(), entityInputs); @@ -183,16 +197,4 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat } } - protected JsonNode createResultJson(boolean useLatestTs, JsonNode result) { - long latestTs = getLatestTimestamp(); - if (useLatestTs && latestTs != -1) { - ObjectNode resultNode = JacksonUtil.newObjectNode(); - resultNode.put("ts", latestTs); - resultNode.set("values", result); - return resultNode; - } else { - return result; - } - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java similarity index 72% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java index e002aece2c..5385808923 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java @@ -27,7 +27,7 @@ import java.util.Map; @Data @AllArgsConstructor -public class AggArgumentEntry implements ArgumentEntry { +public class RelatedEntitiesArgumentEntry implements ArgumentEntry { private final Map aggInputs; @@ -35,7 +35,7 @@ public class AggArgumentEntry implements ArgumentEntry { @Override public ArgumentEntryType getType() { - return ArgumentEntryType.AGGREGATE_LATEST; + return ArgumentEntryType.RELATED_ENTITIES; } @Override @@ -45,19 +45,15 @@ public class AggArgumentEntry implements ArgumentEntry { @Override public boolean updateEntry(ArgumentEntry entry) { - if (entry instanceof AggArgumentEntry aggArgumentEntry) { - aggInputs.putAll(aggArgumentEntry.aggInputs); + if (entry instanceof RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry) { + aggInputs.putAll(relatedEntitiesArgumentEntry.aggInputs); return true; } else if (entry instanceof AggSingleEntityArgumentEntry aggSingleEntityArgumentEntry) { - if (aggSingleEntityArgumentEntry.isDeleted()) { - aggInputs.remove(aggSingleEntityArgumentEntry.getEntityId()); + ArgumentEntry argumentEntry = aggInputs.get(aggSingleEntityArgumentEntry.getEntityId()); + if (argumentEntry != null) { + argumentEntry.updateEntry(aggSingleEntityArgumentEntry); } else { - ArgumentEntry argumentEntry = aggInputs.get(aggSingleEntityArgumentEntry.getEntityId()); - if (argumentEntry != null) { - argumentEntry.updateEntry(aggSingleEntityArgumentEntry); - } else { - aggInputs.put(aggSingleEntityArgumentEntry.getEntityId(), aggSingleEntityArgumentEntry); - } + aggInputs.put(aggSingleEntityArgumentEntry.getEntityId(), aggSingleEntityArgumentEntry); } return true; } else { diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index e1763ffa2d..239ffedbc7 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -35,7 +35,7 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.RelaredEntitiesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalculatedFieldState; @@ -91,7 +91,7 @@ public class CalculatedFieldArgumentUtils { case GEOFENCING -> new GeofencingCalculatedFieldState(entityId); case ALARM -> new AlarmCalculatedFieldState(entityId); case PROPAGATION -> new PropagationCalculatedFieldState(entityId); - case RELATED_ENTITIES_AGGREGATION -> new RelaredEntitiesAggregationCalculatedFieldState(entityId); + case RELATED_ENTITIES_AGGREGATION -> new RelatedEntitiesAggregationCalculatedFieldState(entityId); }; } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 67af13fdb3..389fad8ff3 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -47,9 +47,9 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.RelaredEntitiesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; @@ -104,9 +104,9 @@ public class CalculatedFieldUtils { case SINGLE_VALUE -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); case TS_ROLLING -> builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry)); case GEOFENCING -> builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); - case AGGREGATE_LATEST -> { - AggArgumentEntry aggArgumentEntry = (AggArgumentEntry) argEntry; - aggArgumentEntry.getAggInputs() + case RELATED_ENTITIES -> { + RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry; + relatedEntitiesArgumentEntry.getAggInputs() .forEach((entityId, entry) -> aggBuilder.addAggArguments(toAggSingleArgumentProto(argName, entityId, entry))); } } @@ -120,7 +120,7 @@ public class CalculatedFieldUtils { alarmStateProto.setClearRuleState(toAlarmRuleStateProto(alarmState.getClearRuleState())); } } - if (state instanceof RelaredEntitiesAggregationCalculatedFieldState aggState) { + if (state instanceof RelatedEntitiesAggregationCalculatedFieldState aggState) { aggBuilder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); builder.setLatestValuesAggregationState(aggBuilder.build()); } @@ -214,7 +214,7 @@ public class CalculatedFieldUtils { case GEOFENCING -> new GeofencingCalculatedFieldState(id.entityId()); case ALARM -> new AlarmCalculatedFieldState(id.entityId()); case PROPAGATION -> new PropagationCalculatedFieldState(id.entityId()); - case RELATED_ENTITIES_AGGREGATION -> new RelaredEntitiesAggregationCalculatedFieldState(id.entityId()); + case RELATED_ENTITIES_AGGREGATION -> new RelatedEntitiesAggregationCalculatedFieldState(id.entityId()); }; proto.getSingleValueArgumentsList().forEach(argProto -> @@ -241,7 +241,7 @@ public class CalculatedFieldUtils { } } case RELATED_ENTITIES_AGGREGATION -> { - RelaredEntitiesAggregationCalculatedFieldState aggState = (RelaredEntitiesAggregationCalculatedFieldState) state; + RelatedEntitiesAggregationCalculatedFieldState aggState = (RelatedEntitiesAggregationCalculatedFieldState) state; LatestValuesAggregationStateProto aggregationStateProto = proto.getLatestValuesAggregationState(); Map> arguments = new HashMap<>(); aggregationStateProto.getAggArgumentsList().forEach(argProto -> { @@ -249,7 +249,7 @@ public class CalculatedFieldUtils { arguments.computeIfAbsent(argProto.getValue().getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); }); arguments.forEach((argName, entityInputs) -> { - aggState.getArguments().put(argName, new AggArgumentEntry(entityInputs, false)); + aggState.getArguments().put(argName, new RelatedEntitiesArgumentEntry(entityInputs, false)); }); aggState.setLastArgsRefreshTs(aggregationStateProto.getLastArgsUpdateTs()); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java similarity index 80% rename from application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java rename to application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java index c1c2d85c55..14d2801e0e 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java @@ -21,7 +21,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import java.util.HashMap; @@ -31,9 +31,9 @@ import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -public class AggArgumentEntryTest { +public class RelatedEntitiesArgumentEntryTest { - private AggArgumentEntry entry; + private RelatedEntitiesArgumentEntry entry; private final DeviceId device1 = new DeviceId(UUID.fromString("1984e5f4-9ff0-4187-84ae-e4438bba4c8a")); private final DeviceId device2 = new DeviceId(UUID.fromString("937fc062-1a9d-438f-aa22-55a93fc908b7")); @@ -46,7 +46,7 @@ public class AggArgumentEntryTest { aggInputs.put(device1, new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 12L), 1L))); aggInputs.put(device2, new AggSingleEntityArgumentEntry(device2, new BasicTsKvEntry(ts - 150, new LongDataEntry("key", 16L), 6L))); - entry = new AggArgumentEntry(aggInputs, false); + entry = new RelatedEntitiesArgumentEntry(aggInputs, false); } @Test @@ -61,17 +61,17 @@ public class AggArgumentEntryTest { DeviceId device3 = new DeviceId(UUID.randomUUID()); DeviceId device4 = new DeviceId(UUID.randomUUID()); - AggArgumentEntry aggArgumentEntry = new AggArgumentEntry(Map.of( + RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = new RelatedEntitiesArgumentEntry(Map.of( device3, new AggSingleEntityArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 16L), 13L)), device4, new AggSingleEntityArgumentEntry(device4, new BasicTsKvEntry(ts - 60, new LongDataEntry("key", 23L), 7L)) ), false); - assertThat(entry.updateEntry(aggArgumentEntry)).isTrue(); + assertThat(entry.updateEntry(relatedEntitiesArgumentEntry)).isTrue(); Map aggInputs = entry.getAggInputs(); assertThat(aggInputs.size()).isEqualTo(4); - assertThat(aggInputs.get(device3)).isEqualTo(aggArgumentEntry.getAggInputs().get(device3)); - assertThat(aggInputs.get(device4)).isEqualTo(aggArgumentEntry.getAggInputs().get(device4)); + assertThat(aggInputs.get(device3)).isEqualTo(relatedEntitiesArgumentEntry.getAggInputs().get(device3)); + assertThat(aggInputs.get(device4)).isEqualTo(relatedEntitiesArgumentEntry.getAggInputs().get(device4)); } @Test @@ -98,15 +98,4 @@ public class AggArgumentEntryTest { assertThat(aggInputs.get(device2)).isEqualTo(singleEntityArgumentEntry); } - @Test - void testUpdateEntryWhenDeletedAggSingleEntityArgumentEntryPassed() { - AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device2, true); - - assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); - - Map aggInputs = entry.getAggInputs(); - assertThat(aggInputs.size()).isEqualTo(1); - assertThat(aggInputs.get(device2)).isNull(); - } - } From 7bf7e0f994e4d69c5ca2f1f8e7464021c9310f48 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Oct 2025 15:33:02 +0300 Subject: [PATCH 14/33] removed unnecessary fetch arguments methods --- ...tractCalculatedFieldProcessingService.java | 77 ++++++------------- ...faultCalculatedFieldProcessingService.java | 3 - .../service/cf/ctx/state/ArgumentEntry.java | 8 +- .../utils/CalculatedFieldArgumentUtils.java | 9 --- 4 files changed, 27 insertions(+), 70 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 0c0f2f23e9..cfd7c9af3e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -64,7 +64,6 @@ import static org.thingsboard.server.common.data.cf.configuration.geofencing.Ent import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LONGITUDE_ARGUMENT_KEY; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultAttributeEntry; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultKvEntry; -import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformAggSingleArgument; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformSingleValueArgument; @Data @@ -98,7 +97,7 @@ public abstract class AbstractCalculatedFieldProcessingService { Map> argFutures = switch (ctx.getCfType()) { case GEOFENCING -> fetchGeofencingCalculatedFieldArguments(ctx, entityId, false, ts); case SIMPLE, SCRIPT, ALARM, PROPAGATION -> getBaseCalculatedFieldArguments(ctx, entityId, ts); - case RELATED_ENTITIES_AGGREGATION -> fetchAggArguments(ctx, entityId, ts); + case RELATED_ENTITIES_AGGREGATION -> fetchRelatedEntitiesAggArguments(ctx, entityId, ts); }; if (ctx.getCfType() == PROPAGATION) { argFutures.put(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId)); @@ -128,23 +127,6 @@ public abstract class AbstractCalculatedFieldProcessingService { return resolveOwnerArgument(tenantId, entityId); } - private ListenableFuture> resolveRelatedEntities(TenantId tenantId, EntityId entityId, RelationPathLevel relation) { - ListenableFuture> relationsFut = relationService.findByRelationPathQueryAsync(tenantId, new EntityRelationPathQuery(entityId, List.of(relation))); - - return Futures.transform(relationsFut, relations -> { - if (relations == null) { - return new ArrayList<>(); - } - - return switch (relation.direction()) { - case FROM -> relations.stream() - .map(EntityRelation::getTo) - .toList(); - case TO -> relations.isEmpty() ? List.of() : List.of(relations.get(0).getFrom()); - }; - }, calculatedFieldCallbackExecutor); - } - protected Map resolveArgumentFutures(Map> argFutures) { return argFutures.entrySet().stream() .collect(Collectors.toMap( @@ -189,7 +171,7 @@ public abstract class AbstractCalculatedFieldProcessingService { return argFutures; } - protected Map> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { + protected Map> fetchRelatedEntitiesAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig = (RelatedEntitiesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); ListenableFuture> relatedEntitiesFut = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getRelation()); @@ -197,10 +179,27 @@ public abstract class AbstractCalculatedFieldProcessingService { return aggConfig.getArguments().entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, - entry -> Futures.transformAsync(relatedEntitiesFut, relatedEntities -> fetchAggArgumentEntry(ctx.getTenantId(), relatedEntities, entry.getValue(), ts), MoreExecutors.directExecutor()) + entry -> Futures.transformAsync(relatedEntitiesFut, relatedEntities -> fetchRelatedEntitiesArgumentEntry(ctx.getTenantId(), relatedEntities, entry.getValue(), ts), MoreExecutors.directExecutor()) )); } + private ListenableFuture> resolveRelatedEntities(TenantId tenantId, EntityId entityId, RelationPathLevel relation) { + ListenableFuture> relationsFut = relationService.findByRelationPathQueryAsync(tenantId, new EntityRelationPathQuery(entityId, List.of(relation))); + + return Futures.transform(relationsFut, relations -> { + if (relations == null) { + return new ArrayList<>(); + } + + return switch (relation.direction()) { + case FROM -> relations.stream() + .map(EntityRelation::getTo) + .toList(); + case TO -> relations.isEmpty() ? List.of() : List.of(relations.get(0).getFrom()); + }; + }, calculatedFieldCallbackExecutor); + } + private ListenableFuture> resolveGeofencingEntityIds(TenantId tenantId, EntityId entityId, Map.Entry entry) { Argument value = entry.getValue(); if (value.getRefEntityId() != null) { @@ -252,11 +251,11 @@ public abstract class AbstractCalculatedFieldProcessingService { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))), MoreExecutors.directExecutor()); } - public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) { + public ListenableFuture fetchRelatedEntitiesArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) { List>> futures = aggEntities.stream() .map(entityId -> { - ListenableFuture singleAggEntryFut = fetchSingleAggArgumentEntry(tenantId, entityId, argument, startTs); - return Futures.transform(singleAggEntryFut, singleAggEntry -> Map.entry(entityId, singleAggEntry), MoreExecutors.directExecutor()); + ListenableFuture argumentEntryFut = fetchArgumentValue(tenantId, entityId, argument, startTs); + return Futures.transform(argumentEntryFut, argumentEntry -> Map.entry(entityId, ArgumentEntry.createAggSingleArgument(entityId, argumentEntry)), MoreExecutors.directExecutor()); }) .toList(); @@ -321,34 +320,4 @@ public abstract class AbstractCalculatedFieldProcessingService { return new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, endTs, 0, limit, Aggregation.NONE); } - private ListenableFuture fetchSingleAggArgumentEntry(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { - return switch (argument.getRefEntityKey().getType()) { - case TS_ROLLING -> throw new IllegalStateException("TS_ROLLING is not supported for aggregation"); - case ATTRIBUTE -> fetchAttributeAggEntry(tenantId, entityId, argument, startTs); - case TS_LATEST -> fetchTsLatestAggEntry(tenantId, entityId, argument, startTs); - }; - } - - private ListenableFuture fetchAttributeAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultLastUpdateTs) { - log.trace("[{}][{}] Fetching attribute for key {}", tenantId, entityId, argument.getRefEntityKey()); - var attributeOptFuture = attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()); - return Futures.transform(attributeOptFuture, attrOpt -> { - log.debug("[{}][{}] Fetched attribute for key {}: {}", tenantId, entityId, argument.getRefEntityKey(), attrOpt); - AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, SingleValueArgumentEntry.DEFAULT_VERSION)); - return transformAggSingleArgument(entityId, Optional.of(attributeKvEntry)); - }, calculatedFieldCallbackExecutor); - } - - private ListenableFuture fetchTsLatestAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultTs) { - String key = argument.getRefEntityKey().getKey(); - log.trace("[{}][{}] Fetching latest timeseries {}", tenantId, entityId, key); - return Futures.transform( - timeseriesService.findLatest(tenantId, entityId, key), - result -> { - log.debug("[{}][{}] Fetched latest timeseries {}: {}", tenantId, entityId, key, result); - Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(defaultTs, createDefaultKvEntry(argument), SingleValueArgumentEntry.DEFAULT_VERSION))); - return transformAggSingleArgument(entityId, tsKvEntry); - }, calculatedFieldCallbackExecutor); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index d7957dce9b..52393d0ffe 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.service.cf; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; @@ -56,7 +54,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.stream.Collectors; import static org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration.PROPAGATION_CONFIG_ARGUMENT; import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 5bb16292be..863d6f5b50 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -22,8 +22,8 @@ import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry; @@ -40,7 +40,7 @@ import java.util.Map; @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING"), @JsonSubTypes.Type(value = PropagationArgumentEntry.class, name = "PROPAGATION"), - @JsonSubTypes.Type(value = RelatedEntitiesArgumentEntry.class, name = "AGGREGATE_LATEST"), + @JsonSubTypes.Type(value = RelatedEntitiesArgumentEntry.class, name = "RELATED_ENTITIES"), @JsonSubTypes.Type(value = AggSingleEntityArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") }) public interface ArgumentEntry { @@ -80,8 +80,8 @@ public interface ArgumentEntry { return new RelatedEntitiesArgumentEntry(entityIdkvEntryMap, false); } - static ArgumentEntry createAggSingleArgument(EntityId entityId, KvEntry kvEntry) { - return new AggSingleEntityArgumentEntry(entityId, kvEntry); + static ArgumentEntry createAggSingleArgument(EntityId entityId, ArgumentEntry argumentEntry) { + return new AggSingleEntityArgumentEntry(entityId, argumentEntry); } } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index 239ffedbc7..0c0d401688 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -34,7 +34,6 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -56,14 +55,6 @@ public class CalculatedFieldArgumentUtils { } } - public static ArgumentEntry transformAggSingleArgument(EntityId entityId, Optional kvEntry) { - if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { - return ArgumentEntry.createAggSingleArgument(entityId, kvEntry.get()); - } else { - return new AggSingleEntityArgumentEntry(); - } - } - public static KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getRefEntityKey().getKey(); String defaultValue = argument.getDefaultValue(); From 899dd9002bb045042ca439566eefa4efe55bd4d3 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Oct 2025 17:00:55 +0300 Subject: [PATCH 15/33] removed single agg argument and updated old single value argument --- ...CalculatedFieldEntityMessageProcessor.java | 15 ++- ...tractCalculatedFieldProcessingService.java | 2 +- .../service/cf/ctx/state/ArgumentEntry.java | 12 +-- .../cf/ctx/state/ArgumentEntryType.java | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 15 +-- .../cf/ctx/state/CalculatedFieldCtx.java | 2 +- .../cf/ctx/state/CalculatedFieldState.java | 1 - .../ctx/state/SimpleCalculatedFieldState.java | 2 +- .../ctx/state/SingleValueArgumentEntry.java | 30 ++++++ .../AggSingleEntityArgumentEntry.java | 97 ----------------- ...titiesAggregationCalculatedFieldState.java | 16 +-- .../RelatedEntitiesArgumentEntry.java | 13 +-- .../state/aggregation/function/AggEntry.java | 15 ++- .../function/AggFunctionFactory.java | 33 ------ .../aggregation/function/AvgAggEntry.java | 6 +- .../aggregation/function/BaseAggEntry.java | 9 +- .../aggregation/function/CountAggEntry.java | 2 +- .../function/CountUniqueAggEntry.java | 2 +- .../aggregation/function/MaxAggEntry.java | 5 +- .../aggregation/function/MinAggEntry.java | 5 +- .../aggregation/function/SumAggEntry.java | 5 +- .../server/utils/CalculatedFieldUtils.java | 61 ++++------- .../AggSingleEntityArgumentEntryTest.java | 101 ------------------ .../RelatedEntitiesArgumentEntryTest.java | 17 ++- .../configuration/aggregation/AggInput.java | 2 + ...gregationCalculatedFieldConfiguration.java | 12 +-- common/proto/src/main/proto/queue.proto | 14 +-- .../thingsboard/script/api/tbel/TbUtils.java | 11 ++ .../script/api/tbel/TbelCfArg.java | 2 +- ... => TbelCfRelatedEntitiesAggregation.java} | 4 +- 30 files changed, 148 insertions(+), 365 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java delete mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java rename common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/{TbelCfLatestValuesAggregation.java => TbelCfRelatedEntitiesAggregation.java} (90%) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 434f555325..ca97dae51b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -52,7 +52,6 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; @@ -232,7 +231,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else { if (state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, msg.getRelatedEntityId(), ctx.getArguments()); - updatedArgs = relatedEntitiesAggState.updateEntityData(toAggSingleEntityArguments(msg.getRelatedEntityId(), fetchedArgs)); + updatedArgs = relatedEntitiesAggState.updateEntityData(setEntityIdToSingleEntityArguments(msg.getRelatedEntityId(), fetchedArgs)); } state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); @@ -544,7 +543,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); String argName = relatedEntityArgs.get(key); if (argName != null) { - arguments.put(argName, new AggSingleEntityArgumentEntry(originator, item)); + arguments.put(argName, new SingleValueArgumentEntry(originator, item)); } } } @@ -597,7 +596,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); String argName = relatedEntityArgs.get(key); if (argName != null) { - arguments.put(argName, new AggSingleEntityArgumentEntry(entityId, item)); + arguments.put(argName, new SingleValueArgumentEntry(entityId, item)); } } } @@ -636,7 +635,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM SingleValueArgumentEntry argumentEntry = StringUtils.isNotEmpty(defaultValue) ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) : new SingleValueArgumentEntry(); - arguments.put(argName, new AggSingleEntityArgumentEntry(msgEntityId, argumentEntry)); + arguments.put(argName, new SingleValueArgumentEntry(msgEntityId, argumentEntry)); } } } @@ -668,18 +667,18 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); if (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION.equals(ctx.getCfType())) { - fetchedArgs = toAggSingleEntityArguments(entityId, fetchedArgs); + fetchedArgs = setEntityIdToSingleEntityArguments(entityId, fetchedArgs); } fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); return fetchedArgs; } - private Map toAggSingleEntityArguments(EntityId relatedEntityId, Map fetchedArgs) { + private Map setEntityIdToSingleEntityArguments(EntityId relatedEntityId, Map fetchedArgs) { return fetchedArgs.entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, - argEntry -> new AggSingleEntityArgumentEntry(relatedEntityId, argEntry.getValue()) + argEntry -> new SingleValueArgumentEntry(relatedEntityId, argEntry.getValue()) )); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index cfd7c9af3e..d4bf0d52be 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -255,7 +255,7 @@ public abstract class AbstractCalculatedFieldProcessingService { List>> futures = aggEntities.stream() .map(entityId -> { ListenableFuture argumentEntryFut = fetchArgumentValue(tenantId, entityId, argument, startTs); - return Futures.transform(argumentEntryFut, argumentEntry -> Map.entry(entityId, ArgumentEntry.createAggSingleArgument(entityId, argumentEntry)), MoreExecutors.directExecutor()); + return Futures.transform(argumentEntryFut, argumentEntry -> Map.entry(entityId, ArgumentEntry.createSingleValueArgument(entityId, argumentEntry)), MoreExecutors.directExecutor()); }) .toList(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 863d6f5b50..b331c11a47 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -22,7 +22,6 @@ import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry; @@ -40,8 +39,7 @@ import java.util.Map; @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING"), @JsonSubTypes.Type(value = PropagationArgumentEntry.class, name = "PROPAGATION"), - @JsonSubTypes.Type(value = RelatedEntitiesArgumentEntry.class, name = "RELATED_ENTITIES"), - @JsonSubTypes.Type(value = AggSingleEntityArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") + @JsonSubTypes.Type(value = RelatedEntitiesArgumentEntry.class, name = "RELATED_ENTITIES") }) public interface ArgumentEntry { @@ -64,6 +62,10 @@ public interface ArgumentEntry { return new SingleValueArgumentEntry(kvEntry); } + static ArgumentEntry createSingleValueArgument(EntityId entityId, ArgumentEntry argumentEntry) { + return new SingleValueArgumentEntry(entityId, argumentEntry); + } + static ArgumentEntry createTsRollingArgument(List kvEntries, int limit, long timeWindow) { return new TsRollingArgumentEntry(kvEntries, limit, timeWindow); } @@ -80,8 +82,4 @@ public interface ArgumentEntry { return new RelatedEntitiesArgumentEntry(entityIdkvEntryMap, false); } - static ArgumentEntry createAggSingleArgument(EntityId entityId, ArgumentEntry argumentEntry) { - return new AggSingleEntityArgumentEntry(entityId, argumentEntry); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java index 5c672cf04e..427df2bf5b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java @@ -16,5 +16,5 @@ package org.thingsboard.server.service.cf.ctx.state; public enum ArgumentEntryType { - SINGLE_VALUE, TS_ROLLING, GEOFENCING, PROPAGATION, RELATED_ENTITIES, AGGREGATE_LATEST_SINGLE + SINGLE_VALUE, TS_ROLLING, GEOFENCING, PROPAGATION, RELATED_ENTITIES } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 91c331f88e..d48ed9c268 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -18,13 +18,12 @@ package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Getter; import lombok.Setter; -import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.TbActorRef; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.utils.CalculatedFieldUtils; import java.io.Closeable; @@ -76,7 +75,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, ArgumentEntry existingEntry = arguments.get(key); boolean entryUpdated; - if (existingEntry == null || !(newEntry instanceof AggSingleEntityArgumentEntry) && newEntry.isForceResetPrevious()) { + if (existingEntry == null || !ctx.getCfType().equals(CalculatedFieldType.RELATED_ENTITIES_AGGREGATION) && newEntry.isForceResetPrevious()) { validateNewEntry(key, newEntry); arguments.put(key, newEntry); entryUpdated = true; @@ -152,14 +151,4 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, this.latestTimestamp = Math.max(this.latestTimestamp, newTs); } - protected Object formatResult(double result, Integer decimals) { - if (decimals == null) { - return result; - } - if (decimals.equals(0)) { - return TbUtils.toInt(result); - } - return TbUtils.toFixed(result, decimals); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 46442847a0..aab858d85d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -634,7 +634,7 @@ public class CalculatedFieldCtx { private boolean hasRelatedEntitiesAggregationConfigurationChanges(CalculatedFieldCtx other) { if (calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration thisConfig && other.calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration otherConfig) { - return !thisConfig.getArguments().equals(otherConfig.getArguments()) || !thisConfig.getRelation().equals(otherConfig.getRelation()); + return !thisConfig.getRelation().equals(otherConfig.getRelation()); } return false; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 8e202308d0..2b3ba19528 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -33,7 +33,6 @@ import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalcul import java.io.Closeable; import java.util.Map; -import java.util.concurrent.ExecutionException; import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArgumentProto; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index a268577d4b..ab0ed26dfe 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -52,7 +52,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { double expressionResult = ctx.evaluateSimpleExpression(expression.get(), this); Output output = ctx.getOutput(); - Object result = formatResult(expressionResult, output.getDecimalsByDefault()); + Object result = TbUtils.roundResult(expressionResult, output.getDecimalsByDefault()); JsonNode outputResult = createResultJson(ctx.isUseLatestTs(), output.getName(), result); return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index e81201c961..67fb385917 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -20,9 +20,11 @@ import com.fasterxml.jackson.core.type.TypeReference; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import org.springframework.lang.Nullable; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; @@ -37,6 +39,9 @@ import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; @AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { + @Nullable + protected EntityId entityId; + protected long ts; protected BasicKvEntry kvEntryValue; protected Long version; @@ -45,6 +50,11 @@ public class SingleValueArgumentEntry implements ArgumentEntry { public static final Long DEFAULT_VERSION = -1L; + public SingleValueArgumentEntry(EntityId entityId, ArgumentEntry entry) { + this(entry); + this.entityId = entityId; + } + public SingleValueArgumentEntry(ArgumentEntry entry) { if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { this.ts = singleValueArgumentEntry.ts; @@ -54,6 +64,11 @@ public class SingleValueArgumentEntry implements ArgumentEntry { } } + public SingleValueArgumentEntry(EntityId entityId, TsKvProto entry) { + this(entry); + this.entityId = entityId; + } + public SingleValueArgumentEntry(TsKvProto entry) { this.ts = entry.getTs(); if (entry.hasVersion()) { @@ -62,6 +77,11 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.kvEntryValue = ProtoUtils.fromProto(entry.getKv()); } + public SingleValueArgumentEntry(EntityId entityId, AttributeValueProto entry) { + this(entry); + this.entityId = entityId; + } + public SingleValueArgumentEntry(AttributeValueProto entry) { this.ts = entry.getLastUpdateTs(); if (entry.hasVersion()) { @@ -70,6 +90,11 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.kvEntryValue = ProtoUtils.basicKvEntryFromProto(entry); } + public SingleValueArgumentEntry(EntityId entityId, KvEntry entry) { + this(entry); + this.entityId = entityId; + } + public SingleValueArgumentEntry(KvEntry entry) { if (entry instanceof TsKvEntry tsKvEntry) { this.ts = tsKvEntry.getTs(); @@ -81,6 +106,11 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.kvEntryValue = ProtoUtils.basicKvEntryFromKvEntry(entry); } + public SingleValueArgumentEntry(EntityId entityId, long ts, BasicKvEntry kvEntryValue, Long version) { + this(ts, kvEntryValue, version); + this.entityId = entityId; + } + public SingleValueArgumentEntry(long ts, BasicKvEntry kvEntryValue, Long version) { this.ts = ts; this.kvEntryValue = kvEntryValue; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java deleted file mode 100644 index b935256860..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf.ctx.state.aggregation; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.kv.BasicKvEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; -import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; -import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class AggSingleEntityArgumentEntry extends SingleValueArgumentEntry { - - private EntityId entityId; - - public AggSingleEntityArgumentEntry(EntityId entityId, ArgumentEntry entry) { - super(entry); - this.entityId = entityId; - } - - public AggSingleEntityArgumentEntry(EntityId entityId, TsKvProto entry) { - super(entry); - this.entityId = entityId; - } - - public AggSingleEntityArgumentEntry(EntityId entityId, AttributeValueProto entry) { - super(entry); - this.entityId = entityId; - } - - public AggSingleEntityArgumentEntry(EntityId entityId, KvEntry entry) { - super(entry); - this.entityId = entityId; - } - - public AggSingleEntityArgumentEntry(EntityId entityId, long ts, BasicKvEntry kvEntryValue, Long version) { - super(ts, kvEntryValue, version); - this.entityId = entityId; - } - - @Override - public boolean updateEntry(ArgumentEntry entry) { - if (entry instanceof AggSingleEntityArgumentEntry aggSingleEntityEntry) { - if (aggSingleEntityEntry.isForceResetPrevious()) { - return applyNewEntry(aggSingleEntityEntry); - } - - if (aggSingleEntityEntry.getTs() < this.ts) { - if (!isDefaultValue()) { - return false; - } - } - - Long newVersion = aggSingleEntityEntry.getVersion(); - if (newVersion == null || this.version == null || newVersion > this.version) { - return applyNewEntry(aggSingleEntityEntry); - } - } else { - throw new IllegalArgumentException("Unsupported argument entry type for aggregation single entity argument entry: " + entry.getType()); - } - return false; - } - - private boolean applyNewEntry(AggSingleEntityArgumentEntry entry) { - this.ts = entry.getTs(); - this.version = entry.getVersion(); - this.kvEntryValue = entry.getKvEntryValue(); - this.entityId = entry.getEntityId(); - return true; - } - - @Override - public ArgumentEntryType getType() { - return ArgumentEntryType.AGGREGATE_LATEST_SINGLE; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java index c0baca836c..8e78824c7c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java @@ -37,7 +37,6 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.BaseCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.aggregation.function.AggEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.function.AggFunctionFactory; import java.util.HashMap; import java.util.Map; @@ -150,10 +149,10 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat String metricKey = entry.getKey(); AggMetric metric = entry.getValue(); - AggEntry aggMetricEntry = AggFunctionFactory.createAggFunction(metric.getFunction()); + AggEntry aggMetricEntry = AggEntry.createAggFunction(metric.getFunction()); aggregateMetric(metric, aggMetricEntry, inputs); - aggMetricEntry.result().ifPresent(result -> { - aggResult.set(metricKey, JacksonUtil.valueToTree(formatResult(result, output.getDecimalsByDefault()))); + aggMetricEntry.result(output.getDecimalsByDefault()).ifPresent(result -> { + aggResult.set(metricKey, JacksonUtil.valueToTree(result)); }); } return aggResult; @@ -188,13 +187,4 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat } } - private Object formatResult(Object aggregationResult, Integer decimals) { - try { - double result = Double.parseDouble(aggregationResult.toString()); - return formatResult(result, decimals); - } catch (Exception e) { - throw new IllegalArgumentException("Aggregation result cannot be parsed: " + aggregationResult, e); - } - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java index 5385808923..45b7755af4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java @@ -18,10 +18,11 @@ package org.thingsboard.server.service.cf.ctx.state.aggregation; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.script.api.tbel.TbelCfArg; -import org.thingsboard.script.api.tbel.TbelCfLatestValuesAggregation; +import org.thingsboard.script.api.tbel.TbelCfRelatedEntitiesAggregation; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import java.util.Map; @@ -48,12 +49,12 @@ public class RelatedEntitiesArgumentEntry implements ArgumentEntry { if (entry instanceof RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry) { aggInputs.putAll(relatedEntitiesArgumentEntry.aggInputs); return true; - } else if (entry instanceof AggSingleEntityArgumentEntry aggSingleEntityArgumentEntry) { - ArgumentEntry argumentEntry = aggInputs.get(aggSingleEntityArgumentEntry.getEntityId()); + } else if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + ArgumentEntry argumentEntry = aggInputs.get(singleValueArgumentEntry.getEntityId()); if (argumentEntry != null) { - argumentEntry.updateEntry(aggSingleEntityArgumentEntry); + argumentEntry.updateEntry(singleValueArgumentEntry); } else { - aggInputs.put(aggSingleEntityArgumentEntry.getEntityId(), aggSingleEntityArgumentEntry); + aggInputs.put(singleValueArgumentEntry.getEntityId(), singleValueArgumentEntry); } return true; } else { @@ -68,7 +69,7 @@ public class RelatedEntitiesArgumentEntry implements ArgumentEntry { @Override public TbelCfArg toTbelCfArg() { - return new TbelCfLatestValuesAggregation(aggInputs.values()); + return new TbelCfRelatedEntitiesAggregation(aggInputs.values()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java index 4239e94fec..c4b93fd91d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation.function; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; @@ -36,10 +37,22 @@ import java.util.Optional; }) public interface AggEntry { + @JsonIgnore AggFunction getType(); void update(Object value); - Optional result(); + Optional result(Integer precision); + + static AggEntry createAggFunction(AggFunction function) { + return switch (function) { + case MIN -> new MinAggEntry(); + case MAX -> new MaxAggEntry(); + case SUM -> new SumAggEntry(); + case AVG -> new AvgAggEntry(); + case COUNT -> new CountAggEntry(); + case COUNT_UNIQUE -> new CountUniqueAggEntry(); + }; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java deleted file mode 100644 index 5ccc355b1f..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf.ctx.state.aggregation.function; - -import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; - -public class AggFunctionFactory { - - public static AggEntry createAggFunction(AggFunction function) { - return switch (function) { - case MIN -> new MinAggEntry(); - case MAX -> new MaxAggEntry(); - case SUM -> new SumAggEntry(); - case AVG -> new AvgAggEntry(); - case COUNT -> new CountAggEntry(); - case COUNT_UNIQUE -> new CountUniqueAggEntry(); - }; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java index afe6abb93e..e063ff2ea2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation.function; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; import java.math.BigDecimal; @@ -34,8 +35,9 @@ public class AvgAggEntry extends BaseAggEntry { } @Override - protected double prepareResult() { - return sum.divide(BigDecimal.valueOf(count), 10, RoundingMode.HALF_UP).doubleValue(); + protected Object prepareResult(Integer precision) { + double result = sum.divide(BigDecimal.valueOf(count), RoundingMode.HALF_UP).doubleValue(); + return TbUtils.roundResult(result, precision); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java index 0c12bd13c0..b320435e99 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java @@ -28,10 +28,10 @@ public abstract class BaseAggEntry implements AggEntry { } @Override - public Optional result() { + public Optional result(Integer precision) { if (hasResult) { hasResult = false; - return Optional.of(prepareResult()); + return Optional.of(prepareResult(precision)); } else { return Optional.empty(); } @@ -39,10 +39,13 @@ public abstract class BaseAggEntry implements AggEntry { protected abstract void doUpdate(double value); - protected abstract double prepareResult(); + protected abstract Object prepareResult(Integer precision); protected double extractDoubleValue(Object value) { try { + if (value instanceof Number) { + return ((Number) value).doubleValue(); + } return Double.parseDouble(value.toString()); } catch (Exception e) { throw new NumberFormatException("Cannot parse value " + value.toString()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java index 469048d62d..09116985d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java @@ -29,7 +29,7 @@ public class CountAggEntry implements AggEntry { } @Override - public Optional result() { + public Optional result(Integer precision) { return Optional.of(count); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java index b8b0b92470..efb4a58c90 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java @@ -34,7 +34,7 @@ public class CountUniqueAggEntry implements AggEntry { } @Override - public Optional result() { + public Optional result(Integer precision) { return Optional.of(items.size()); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java index 6e4235b72f..ddc47daf33 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation.function; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; public class MaxAggEntry extends BaseAggEntry { @@ -29,8 +30,8 @@ public class MaxAggEntry extends BaseAggEntry { } @Override - protected double prepareResult() { - return max; + protected Object prepareResult(Integer precision) { + return TbUtils.roundResult(max, precision); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java index eeacc3a7d9..e517ad305f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation.function; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; public class MinAggEntry extends BaseAggEntry { @@ -29,8 +30,8 @@ public class MinAggEntry extends BaseAggEntry { } @Override - protected double prepareResult() { - return min; + protected Object prepareResult(Integer precision) { + return TbUtils.roundResult(min, precision); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java index b90817d784..fe29d27b7e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation.function; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; import java.math.BigDecimal; @@ -31,8 +32,8 @@ public class SumAggEntry extends BaseAggEntry { } @Override - protected double prepareResult() { - return sum.doubleValue(); + protected Object prepareResult(Integer precision) { + return TbUtils.roundResult(sum.doubleValue(), precision); } @Override diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 389fad8ff3..d75ec8a70a 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.util.KvProtoUtil; import org.thingsboard.server.common.util.ProtoUtils; -import org.thingsboard.server.gen.transport.TransportProtos.AggSingleArgumentEntryProto; import org.thingsboard.server.gen.transport.TransportProtos.AlarmRuleStateProto; import org.thingsboard.server.gen.transport.TransportProtos.AlarmStateProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; @@ -35,7 +34,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdPro import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingZoneProto; -import org.thingsboard.server.gen.transport.TransportProtos.LatestValuesAggregationStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.RelatedEntitiesAggregationStateProto; import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.TsDoubleValProto; import org.thingsboard.server.gen.transport.TransportProtos.TsRollingArgumentProto; @@ -47,9 +46,8 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; @@ -98,7 +96,7 @@ public class CalculatedFieldUtils { .setId(toProto(stateId)) .setType(state.getType().name()); - LatestValuesAggregationStateProto.Builder aggBuilder = LatestValuesAggregationStateProto.newBuilder(); + RelatedEntitiesAggregationStateProto.Builder aggBuilder = RelatedEntitiesAggregationStateProto.newBuilder(); state.getArguments().forEach((argName, argEntry) -> { switch (argEntry.getType()) { case SINGLE_VALUE -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); @@ -107,7 +105,7 @@ public class CalculatedFieldUtils { case RELATED_ENTITIES -> { RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry; relatedEntitiesArgumentEntry.getAggInputs() - .forEach((entityId, entry) -> aggBuilder.addAggArguments(toAggSingleArgumentProto(argName, entityId, entry))); + .forEach((entityId, entry) -> aggBuilder.addAggArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) entry))); } } }); @@ -122,7 +120,7 @@ public class CalculatedFieldUtils { } if (state instanceof RelatedEntitiesAggregationCalculatedFieldState aggState) { aggBuilder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); - builder.setLatestValuesAggregationState(aggBuilder.build()); + builder.setRelatedEntitiesAggregationState(aggBuilder.build()); } return builder.build(); } @@ -145,17 +143,6 @@ public class CalculatedFieldUtils { return ruleState; } - public static AggSingleArgumentEntryProto toAggSingleArgumentProto(String argName, EntityId entityId, ArgumentEntry argumentEntry) { - AggSingleArgumentEntryProto.Builder builder = AggSingleArgumentEntryProto.newBuilder() - .setEntityId(ProtoUtils.toProto(entityId)); - - if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - builder.setValue(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); - } - - return builder.build(); - } - public static SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() .setArgName(argName); @@ -166,6 +153,10 @@ public class CalculatedFieldUtils { Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion); + if (entry.getEntityId() != null) { + builder.setEntityId(ProtoUtils.toProto(entry.getEntityId())); + } + return builder.build(); } @@ -242,11 +233,11 @@ public class CalculatedFieldUtils { } case RELATED_ENTITIES_AGGREGATION -> { RelatedEntitiesAggregationCalculatedFieldState aggState = (RelatedEntitiesAggregationCalculatedFieldState) state; - LatestValuesAggregationStateProto aggregationStateProto = proto.getLatestValuesAggregationState(); + RelatedEntitiesAggregationStateProto aggregationStateProto = proto.getRelatedEntitiesAggregationState(); Map> arguments = new HashMap<>(); aggregationStateProto.getAggArgumentsList().forEach(argProto -> { - AggSingleEntityArgumentEntry entry = fromAggSingleValueArgumentProto(argProto); - arguments.computeIfAbsent(argProto.getValue().getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); + SingleValueArgumentEntry entry = fromSingleValueArgumentProto(argProto); + arguments.computeIfAbsent(argProto.getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); }); arguments.forEach((argName, entityInputs) -> { aggState.getArguments().put(argName, new RelatedEntitiesArgumentEntry(entityInputs, false)); @@ -258,31 +249,19 @@ public class CalculatedFieldUtils { return state; } - public static AggSingleEntityArgumentEntry fromAggSingleValueArgumentProto(AggSingleArgumentEntryProto proto) { - if (!proto.hasValue()) { - return new AggSingleEntityArgumentEntry(); - } - EntityId entityId = ProtoUtils.fromProto(proto.getEntityId()); - SingleValueArgumentProto singleValueArgument = proto.getValue(); - TsValueProto tsValueProto = singleValueArgument.getValue(); - return new AggSingleEntityArgumentEntry( - entityId, - tsValueProto.getTs(), - (BasicKvEntry) KvProtoUtil.fromTsValueProto(singleValueArgument.getArgName(), tsValueProto), - singleValueArgument.getVersion() - ); - } - public static SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { if (!proto.hasValue()) { return new SingleValueArgumentEntry(); } TsValueProto tsValueProto = proto.getValue(); - return new SingleValueArgumentEntry( - tsValueProto.getTs(), - (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto), - proto.getVersion() - ); + BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto); + long ts = tsValueProto.getTs(); + long version = proto.getVersion(); + if (proto.hasEntityId()) { + EntityId entityId = ProtoUtils.fromProto(proto.getEntityId()); + return new SingleValueArgumentEntry(entityId, ts, kvEntry, version); + } + return new SingleValueArgumentEntry(ts, kvEntry, version); } public static TsRollingArgumentEntry fromRollingArgumentProto(TsRollingArgumentProto proto) { diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java deleted file mode 100644 index 0401bb0156..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf.ctx.state; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; - -import java.util.UUID; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -public class AggSingleEntityArgumentEntryTest { - - private AggSingleEntityArgumentEntry entry; - - private final DeviceId device1 = new DeviceId(UUID.fromString("1984e5f4-9ff0-4187-84ae-e4438bba4c8a")); - - private final long ts = System.currentTimeMillis(); - - @BeforeEach - void setUp() { - entry = new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 12L), 22L)); - } - - @Test - void testUpdateEntryWhenNotAggEntryPassed() { - assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L))) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Unsupported argument entry type for aggregation single entity argument entry: " + ArgumentEntryType.TS_ROLLING); - } - - @Test - void testUpdateEntryWhenResetPrevious() { - AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 100L)); - singleEntityArgumentEntry.setForceResetPrevious(true); - - assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); - assertThat(entry.getTs()).isEqualTo(singleEntityArgumentEntry.getTs()); - assertThat(entry.getKvEntryValue()).isEqualTo(singleEntityArgumentEntry.getKvEntryValue()); - assertThat(entry.getVersion()).isEqualTo(singleEntityArgumentEntry.getVersion()); - } - - - @Test - void testUpdateEntryWithTheSameTsAndVersion() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 19L), 22L)))).isFalse(); - } - - @Test - void testUpdateEntryWithTheSameTsAndDifferentVersion() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 134L), 23L)))).isTrue(); - } - - @Test - void testUpdateEntryWhenNewVersionIsNull() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 56L), null)))).isTrue(); - assertThat(entry.getValue()).isEqualTo(56L); - assertThat(entry.getVersion()).isNull(); - } - - @Test - void testUpdateEntryWhenNewVersionIsGreaterThanCurrent() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 76L), 23L)))).isTrue(); - assertThat(entry.getValue()).isEqualTo(76L); - assertThat(entry.getVersion()).isEqualTo(23); - } - - @Test - void testUpdateEntryWhenNewVersionIsLessThanCurrent() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 11L), 20L)))).isFalse(); - } - - @Test - void testUpdateEntryWhenValueWasNotChanged() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 18L), 45L)))).isTrue(); - } - - @Test - void testUpdateEntryWithOldTs() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 150, new LongDataEntry("key", 155L), 45L)))).isFalse(); - } - -} diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java index 14d2801e0e..61b45b83c9 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java @@ -22,7 +22,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import java.util.HashMap; import java.util.Map; @@ -43,8 +42,8 @@ public class RelatedEntitiesArgumentEntryTest { @BeforeEach void setUp() { Map aggInputs = new HashMap<>(); - aggInputs.put(device1, new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 12L), 1L))); - aggInputs.put(device2, new AggSingleEntityArgumentEntry(device2, new BasicTsKvEntry(ts - 150, new LongDataEntry("key", 16L), 6L))); + aggInputs.put(device1, new SingleValueArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 12L), 1L))); + aggInputs.put(device2, new SingleValueArgumentEntry(device2, new BasicTsKvEntry(ts - 150, new LongDataEntry("key", 16L), 6L))); entry = new RelatedEntitiesArgumentEntry(aggInputs, false); } @@ -62,8 +61,8 @@ public class RelatedEntitiesArgumentEntryTest { DeviceId device4 = new DeviceId(UUID.randomUUID()); RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = new RelatedEntitiesArgumentEntry(Map.of( - device3, new AggSingleEntityArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 16L), 13L)), - device4, new AggSingleEntityArgumentEntry(device4, new BasicTsKvEntry(ts - 60, new LongDataEntry("key", 23L), 7L)) + device3, new SingleValueArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 16L), 13L)), + device4, new SingleValueArgumentEntry(device4, new BasicTsKvEntry(ts - 60, new LongDataEntry("key", 23L), 7L)) ), false); assertThat(entry.updateEntry(relatedEntitiesArgumentEntry)).isTrue(); @@ -75,10 +74,10 @@ public class RelatedEntitiesArgumentEntryTest { } @Test - void testUpdateEntryWhenAggSingleEntityArgumentEntryPassedAndNoEntriesById() { + void testUpdateEntryWhenSingleValueArgumentEntryPassedAndNoEntriesById() { DeviceId device3 = new DeviceId(UUID.randomUUID()); - AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 10L)); + SingleValueArgumentEntry singleEntityArgumentEntry = new SingleValueArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 10L)); assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); @@ -88,8 +87,8 @@ public class RelatedEntitiesArgumentEntryTest { } @Test - void testUpdateEntryWhenAggSingleEntityArgumentEntryPassedAndEntryByIdExist() { - AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device2, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 10L)); + void testUpdateEntryWhenSingleValueArgumentEntryPassedAndEntryByIdExist() { + SingleValueArgumentEntry singleEntityArgumentEntry = new SingleValueArgumentEntry(device2, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 10L)); assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java index fac988d5d9..06929de81c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -31,6 +32,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonIgnoreProperties(ignoreUnknown = true) public interface AggInput { + @JsonIgnore String getType(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java index 69e4ee7fdf..9d4c7bdaf6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; import lombok.Data; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; @@ -27,9 +30,12 @@ import java.util.Map; @Data public class RelatedEntitiesAggregationCalculatedFieldConfiguration implements ArgumentsBasedCalculatedFieldConfiguration { + @NotNull private RelationPathLevel relation; private Map arguments; private long deduplicationIntervalInSec; + @Valid + @NotEmpty private Map metrics; private Output output; private boolean useLatestTs; @@ -41,9 +47,6 @@ public class RelatedEntitiesAggregationCalculatedFieldConfiguration implements A @Override public void validate() { - if (relation == null) { - throw new IllegalArgumentException("Relation must be specified!"); - } relation.validate(); if (arguments.containsKey("ctx")) { throw new IllegalArgumentException("Argument name 'ctx' is reserved and cannot be used."); @@ -51,9 +54,6 @@ public class RelatedEntitiesAggregationCalculatedFieldConfiguration implements A if (arguments.values().stream().anyMatch(Argument::hasTsRollingArgument)) { throw new IllegalArgumentException("Calculated field with type: '" + getType() + "' doesn't support TS_ROLLING arguments."); } - if (metrics.isEmpty()) { - throw new IllegalArgumentException("Latest value aggregation calculated field must have at least one metric."); - } } } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index b59e01128b..2e3a2387e7 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -888,6 +888,7 @@ message SingleValueArgumentProto { string argName = 1; TsValueProto value = 2; int64 version = 3; + EntityIdProto entityId = 4; } message TsDoubleValProto { @@ -915,14 +916,9 @@ message GeofencingArgumentProto { repeated GeofencingZoneProto zones = 2; } -message AggSingleArgumentEntryProto { - EntityIdProto entityId = 1; - SingleValueArgumentProto value = 2; -} - -message LatestValuesAggregationStateProto { +message RelatedEntitiesAggregationStateProto { int64 lastArgsUpdateTs = 1; - repeated AggSingleArgumentEntryProto aggArguments = 2; + repeated SingleValueArgumentProto aggArguments = 2; } message CalculatedFieldStateProto { @@ -932,7 +928,7 @@ message CalculatedFieldStateProto { repeated TsRollingArgumentProto rollingValueArguments = 4; repeated GeofencingArgumentProto geofencingArguments = 5; AlarmStateProto alarmState = 6; - LatestValuesAggregationStateProto latestValuesAggregationState = 7; + RelatedEntitiesAggregationStateProto relatedEntitiesAggregationState = 7; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. @@ -1303,7 +1299,7 @@ message ComponentLifecycleMsgProto { int64 profileIdLSB = 12; optional string info = 13; bool ownerChanged = 100; - bool relationChanged = 15; + bool relationChanged = 14; } message EdgeEventMsgProto { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 072a17835d..3e677cf269 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -1186,6 +1186,17 @@ public class TbUtils { return BigDecimal.valueOf(value).setScale(0, RoundingMode.HALF_UP).intValue(); } + // todo: register method + public static Object roundResult(double value, Integer precision) { + if (precision == null) { + return value; + } + if (precision.equals(0)) { + return toInt(value); + } + return toFixed(value, precision); + } + public static boolean isNaN(double value) { return Double.isNaN(value); } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java index 856cc4bd39..62d6d3d002 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java @@ -29,7 +29,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = TbelCfTsRollingArg.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = TbelCfGeofencingArg.class, name = "GEOFENCING_CF_ARGUMENT_VALUE"), @JsonSubTypes.Type(value = TbelCfPropagationArg.class, name = "PROPAGATION_CF_ARGUMENT_VALUE"), - @JsonSubTypes.Type(value = TbelCfLatestValuesAggregation.class, name = "LATEST_VALUES_AGGREGATION") + @JsonSubTypes.Type(value = TbelCfRelatedEntitiesAggregation.class, name = "LATEST_VALUES_AGGREGATION") }) public interface TbelCfArg extends TbelCfObject { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java similarity index 90% rename from common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java rename to common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java index 4d1b42aa94..75c17f0a0e 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java @@ -20,12 +20,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; @Data -public class TbelCfLatestValuesAggregation implements TbelCfArg { +public class TbelCfRelatedEntitiesAggregation implements TbelCfArg { private final Object value; @JsonCreator - public TbelCfLatestValuesAggregation( + public TbelCfRelatedEntitiesAggregation( @JsonProperty("value") Object value ) { this.value = value; From d97d932cfbe3bc25705f73b6a1f57af15d1b4419 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Oct 2025 10:10:08 +0300 Subject: [PATCH 16/33] refactoring --- ...alculatedFieldManagerMessageProcessor.java | 33 +++++++------- .../cf/TelemetryCalculatedFieldResult.java | 8 ++-- .../ctx/state/BaseCalculatedFieldState.java | 18 +++++--- .../cf/ctx/state/CalculatedFieldCtx.java | 32 +++++--------- ...titiesAggregationCalculatedFieldState.java | 16 +++---- .../RelatedEntitiesArgumentEntry.java | 4 ++ .../aggregation/function/BaseAggEntry.java | 4 +- .../aggregation/function/MaxAggEntry.java | 2 +- .../queue/DefaultTbClusterService.java | 7 +-- .../server/utils/CalculatedFieldUtils.java | 43 ++++++++++--------- .../data/plugin/ComponentLifecycleEvent.java | 4 +- common/proto/src/main/proto/queue.proto | 9 ++-- .../script/api/tbel/TbelCfArg.java | 2 +- .../TbelCfRelatedEntitiesAggregation.java | 2 +- 14 files changed, 95 insertions(+), 89 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 7f1c7b1925..3d61825ebe 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -187,9 +187,18 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) throws CalculatedFieldException { + var event = msg.getData().getEvent(); + if (msg.getData().isRelationChanged()) { + log.debug("Processing relation [{}] event: ", msg.getData().getEvent()); + switch (event) { + case RELATION_UPDATED -> onRelationUpdated(msg.getData(), msg.getCallback()); + case RELATION_DELETED -> onRelationDeleted(msg.getData(), msg.getCallback()); + default -> msg.getCallback().onSuccess(); + } + return; + } log.debug("Processing entity lifecycle event: [{}] for entity: [{}]", msg.getData().getEvent(), msg.getData().getEntityId()); var entityType = msg.getData().getEntityId().getEntityType(); - var event = msg.getData().getEvent(); switch (entityType) { case CALCULATED_FIELD -> { switch (event) { @@ -280,26 +289,20 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } } else if (msg.isOwnerChanged()) { onEntityOwnerChanged(msg, callback); - } else if (msg.isRelationChanged()) { - onRelationUpdated(msg, callback); } else { callback.onSuccess(); } } private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { - if (msg.isRelationChanged()) { - onRelationDeleted(msg, callback); - } else { - switch (msg.getEntityId().getEntityType()) { - case DEVICE, ASSET -> entityProfileCache.removeEntityId(msg.getEntityId()); - case CUSTOMER -> ownerEntities.remove(msg.getEntityId()); - } - ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); - if (isMyPartition(msg.getEntityId(), callback)) { - log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); - getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); - } + switch (msg.getEntityId().getEntityType()) { + case DEVICE, ASSET -> entityProfileCache.removeEntityId(msg.getEntityId()); + case CUSTOMER -> ownerEntities.remove(msg.getEntityId()); + } + ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); + if (isMyPartition(msg.getEntityId(), callback)) { + log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); + getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/TelemetryCalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/TelemetryCalculatedFieldResult.java index 1ad666eac5..d59ec9cca9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/TelemetryCalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/TelemetryCalculatedFieldResult.java @@ -39,6 +39,8 @@ public final class TelemetryCalculatedFieldResult implements CalculatedFieldResu private final AttributeScope scope; private final JsonNode result; + public static TelemetryCalculatedFieldResult EMPTY = TelemetryCalculatedFieldResult.builder().result(null).build(); + @Override public TbMsg toTbMsg(EntityId entityId, List cfIds) { TbMsgType msgType = switch (type) { @@ -66,9 +68,9 @@ public final class TelemetryCalculatedFieldResult implements CalculatedFieldResu @Override public boolean isEmpty() { return result == null || result.isMissingNode() || result.isNull() || - (result.isObject() && result.isEmpty()) || - (result.isArray() && result.isEmpty()) || - (result.isTextual() && result.asText().isEmpty()); + (result.isObject() && result.isEmpty()) || + (result.isArray() && result.isEmpty()) || + (result.isTextual() && result.asText().isEmpty()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index d48ed9c268..d8f13e6a20 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -20,10 +20,10 @@ import lombok.Getter; import lombok.Setter; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.TbActorRef; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.utils.CalculatedFieldUtils; import java.io.Closeable; @@ -75,9 +75,13 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, ArgumentEntry existingEntry = arguments.get(key); boolean entryUpdated; - if (existingEntry == null || !ctx.getCfType().equals(CalculatedFieldType.RELATED_ENTITIES_AGGREGATION) && newEntry.isForceResetPrevious()) { + if (existingEntry == null || newEntry.isForceResetPrevious()) { validateNewEntry(key, newEntry); - arguments.put(key, newEntry); + if (existingEntry instanceof RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry) { + relatedEntitiesArgumentEntry.updateEntry(newEntry); + } else { + arguments.put(key, newEntry); + } entryUpdated = true; } else { entryUpdated = existingEntry.updateEntry(newEntry); @@ -110,7 +114,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, @Override public boolean isReady() { return arguments.keySet().containsAll(requiredArguments) && - arguments.values().stream().noneMatch(ArgumentEntry::isEmpty); + arguments.values().stream().noneMatch(ArgumentEntry::isEmpty); } @Override @@ -122,9 +126,11 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, } @Override - public void close() {} + public void close() { + } - protected void validateNewEntry(String key, ArgumentEntry newEntry) {} + protected void validateNewEntry(String key, ArgumentEntry newEntry) { + } protected ObjectNode toSimpleResult(boolean useLatestTs, ObjectNode valuesNode) { if (!useLatestTs) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index aab858d85d..40e414920a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -263,33 +263,23 @@ public class CalculatedFieldCtx { } public ListenableFuture evaluateTbelExpression(String expression, CalculatedFieldState state) { - return evaluateTbelExpression(tbelExpressions.get(expression), state); + return evaluateTbelExpression(tbelExpressions.get(expression), state.getArguments(), state.getLatestTimestamp()); } public ListenableFuture evaluateTbelExpression(CalculatedFieldScriptEngine expression, CalculatedFieldState state) { - Map arguments = new LinkedHashMap<>(); - List args = new ArrayList<>(argNames.size() + 1); - args.add(new Object()); // first element is a ctx, but we will set it later; - for (String argName : argNames) { - var arg = toTbelArgument(argName, state); - arguments.put(argName, arg); - if (arg instanceof TbelCfSingleValueArg svArg) { - args.add(svArg.getValue()); - } else { - args.add(arg); - } - } - args.set(0, new TbelCfCtx(arguments, state.getLatestTimestamp())); - - return expression.executeScriptAsync(args.toArray()); + return evaluateTbelExpression(expression, state.getArguments(), state.getLatestTimestamp()); } public ListenableFuture evaluateTbelExpression(String expression, Map entries, long latestTimestamp) { + return evaluateTbelExpression(tbelExpressions.get(expression), entries, latestTimestamp); + } + + public ListenableFuture evaluateTbelExpression(CalculatedFieldScriptEngine expression, Map entries, long latestTimestamp) { Map arguments = new LinkedHashMap<>(); List args = new ArrayList<>(argNames.size() + 1); args.add(new Object()); // first element is a ctx, but we will set it later; for (String argName : argNames) { - var arg = entries.get(argName).toTbelCfArg(); + var arg = toTbelArgument(argName, entries); arguments.put(argName, arg); if (arg instanceof TbelCfSingleValueArg svArg) { args.add(svArg.getValue()); @@ -299,7 +289,7 @@ public class CalculatedFieldCtx { } args.set(0, new TbelCfCtx(arguments, latestTimestamp)); - return tbelExpressions.get(expression).executeScriptAsync(args.toArray()); + return expression.executeScriptAsync(args.toArray()); } public ScheduledFuture scheduleReevaluation(long delayMs, TbActorRef actorCtx) { @@ -308,8 +298,8 @@ public class CalculatedFieldCtx { return systemContext.scheduleMsgWithDelay(actorCtx, new CalculatedFieldReevaluateMsg(tenantId, this), delayMs); } - private TbelCfArg toTbelArgument(String key, CalculatedFieldState state) { - return state.getArguments().get(key).toTbelCfArg(); + private TbelCfArg toTbelArgument(String key, Map arguments) { + return arguments.get(key).toTbelCfArg(); } private void initTbelExpression(String expression) { @@ -658,7 +648,7 @@ public class CalculatedFieldCtx { yield true; } yield geofencingState.getLastDynamicArgumentsRefreshTs() < - System.currentTimeMillis() - scheduledUpdateIntervalMillis; + System.currentTimeMillis() - scheduledUpdateIntervalMillis; } default -> false; }; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java index 8e78824c7c..c8731b71f7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java @@ -42,6 +42,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import static java.util.concurrent.TimeUnit.SECONDS; + @Slf4j @Getter public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculatedFieldState { @@ -50,7 +52,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat private long lastArgsRefreshTs = -1; @Setter private long lastMetricsEvalTs = -1; - private long deduplicationInterval = -1; + private long deduplicationIntervalMs = -1; private Map metrics; public RelatedEntitiesAggregationCalculatedFieldState(EntityId entityId) { @@ -62,7 +64,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat super.setCtx(ctx, actorCtx); var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); metrics = configuration.getMetrics(); - deduplicationInterval = configuration.getDeduplicationIntervalInSec(); + deduplicationIntervalMs = SECONDS.toMillis(configuration.getDeduplicationIntervalInSec()); } @Override @@ -76,7 +78,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat @Override public void init() { super.init(); - ctx.scheduleReevaluation(deduplicationInterval, actorCtx); + ctx.scheduleReevaluation(deduplicationIntervalMs, actorCtx); } @Override @@ -97,16 +99,14 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat Output output = ctx.getOutput(); ObjectNode aggResult = aggregateMetrics(output); lastMetricsEvalTs = System.currentTimeMillis(); - ctx.scheduleReevaluation(deduplicationInterval, actorCtx); + ctx.scheduleReevaluation(deduplicationIntervalMs, actorCtx); return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() .type(output.getType()) .scope(output.getScope()) .result(toSimpleResult(ctx.isUseLatestTs(), aggResult)) .build()); } else { - return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() - .result(null) - .build()); + return Futures.immediateFuture(TelemetryCalculatedFieldResult.EMPTY); } } @@ -125,7 +125,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat } private boolean shouldRecalculate() { - boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - deduplicationInterval; + boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - deduplicationIntervalMs; boolean argsUpdatedDuringInterval = lastArgsRefreshTs > lastMetricsEvalTs; return intervalPassed && argsUpdatedDuringInterval; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java index 45b7755af4..5b97b1ed0a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java @@ -50,6 +50,10 @@ public class RelatedEntitiesArgumentEntry implements ArgumentEntry { aggInputs.putAll(relatedEntitiesArgumentEntry.aggInputs); return true; } else if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + if (entry.isForceResetPrevious()) { + aggInputs.put(singleValueArgumentEntry.getEntityId(), singleValueArgumentEntry); + return true; + } ArgumentEntry argumentEntry = aggInputs.get(singleValueArgumentEntry.getEntityId()); if (argumentEntry != null) { argumentEntry.updateEntry(singleValueArgumentEntry); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java index b320435e99..8ca523938d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java @@ -43,8 +43,8 @@ public abstract class BaseAggEntry implements AggEntry { protected double extractDoubleValue(Object value) { try { - if (value instanceof Number) { - return ((Number) value).doubleValue(); + if (value instanceof Number number) { + return number.doubleValue(); } return Double.parseDouble(value.toString()); } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java index ddc47daf33..6d734a5a08 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java @@ -20,7 +20,7 @@ import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFuncti public class MaxAggEntry extends BaseAggEntry { - private double max = -Double.MAX_VALUE; + private double max = Double.MIN_VALUE; @Override protected void doUpdate(double value) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 22faeaf61b..5b6596c9c6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -741,7 +741,7 @@ public class DefaultTbClusterService implements TbClusterService { .tenantId(tenantId) .entityId(entityRelation.getFrom()) .relationChanged(true) - .event(ComponentLifecycleEvent.UPDATED) + .event(ComponentLifecycleEvent.RELATION_UPDATED) .info(JacksonUtil.valueToTree(entityRelation)) .build(); broadcast(msg); @@ -753,7 +753,7 @@ public class DefaultTbClusterService implements TbClusterService { .tenantId(tenantId) .entityId(entityRelation.getFrom()) .relationChanged(true) - .event(ComponentLifecycleEvent.DELETED) + .event(ComponentLifecycleEvent.RELATION_DELETED) .info(JacksonUtil.valueToTree(entityRelation)) .build(); broadcast(msg); @@ -809,7 +809,8 @@ public class DefaultTbClusterService implements TbClusterService { private void pushDeviceUpdateMessage(TenantId tenantId, EdgeId edgeId, EntityId entityId, EdgeEventActionType action) { log.trace("{} Going to send edge update notification for device actor, device id {}, edge id {}", tenantId, entityId, edgeId); switch (action) { - case ASSIGNED_TO_EDGE -> pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); + case ASSIGNED_TO_EDGE -> + pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); case UNASSIGNED_FROM_EDGE -> { EdgeId relatedEdgeId = findRelatedEdgeIdIfAny(tenantId, entityId); pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), relatedEdgeId), null); diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index d75ec8a70a..ba3aa3fd53 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -34,7 +34,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdPro import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingZoneProto; -import org.thingsboard.server.gen.transport.TransportProtos.RelatedEntitiesAggregationStateProto; import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.TsDoubleValProto; import org.thingsboard.server.gen.transport.TransportProtos.TsRollingArgumentProto; @@ -96,16 +95,18 @@ public class CalculatedFieldUtils { .setId(toProto(stateId)) .setType(state.getType().name()); - RelatedEntitiesAggregationStateProto.Builder aggBuilder = RelatedEntitiesAggregationStateProto.newBuilder(); state.getArguments().forEach((argName, argEntry) -> { switch (argEntry.getType()) { - case SINGLE_VALUE -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); - case TS_ROLLING -> builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry)); - case GEOFENCING -> builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); + case SINGLE_VALUE -> + builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); + case TS_ROLLING -> + builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry)); + case GEOFENCING -> + builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); case RELATED_ENTITIES -> { RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry; relatedEntitiesArgumentEntry.getAggInputs() - .forEach((entityId, entry) -> aggBuilder.addAggArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) entry))); + .forEach((entityId, entry) -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) entry))); } } }); @@ -119,8 +120,7 @@ public class CalculatedFieldUtils { } } if (state instanceof RelatedEntitiesAggregationCalculatedFieldState aggState) { - aggBuilder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); - builder.setRelatedEntitiesAggregationState(aggBuilder.build()); + builder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); } return builder.build(); } @@ -208,6 +208,20 @@ public class CalculatedFieldUtils { case RELATED_ENTITIES_AGGREGATION -> new RelatedEntitiesAggregationCalculatedFieldState(id.entityId()); }; + if (state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { + Map> arguments = new HashMap<>(); + proto.getSingleValueArgumentsList().forEach(argProto -> { + SingleValueArgumentEntry entry = fromSingleValueArgumentProto(argProto); + arguments.computeIfAbsent(argProto.getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); + }); + arguments.forEach((argName, entityInputs) -> { + relatedEntitiesAggState.getArguments().put(argName, new RelatedEntitiesArgumentEntry(entityInputs, false)); + }); + relatedEntitiesAggState.setLastArgsRefreshTs(proto.getLastArgsUpdateTs()); + + return relatedEntitiesAggState; + } + proto.getSingleValueArgumentsList().forEach(argProto -> state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); @@ -231,19 +245,6 @@ public class CalculatedFieldUtils { alarmState.setClearRuleState(fromAlarmRuleStateProto(alarmStateProto.getClearRuleState(), alarmState)); } } - case RELATED_ENTITIES_AGGREGATION -> { - RelatedEntitiesAggregationCalculatedFieldState aggState = (RelatedEntitiesAggregationCalculatedFieldState) state; - RelatedEntitiesAggregationStateProto aggregationStateProto = proto.getRelatedEntitiesAggregationState(); - Map> arguments = new HashMap<>(); - aggregationStateProto.getAggArgumentsList().forEach(argProto -> { - SingleValueArgumentEntry entry = fromSingleValueArgumentProto(argProto); - arguments.computeIfAbsent(argProto.getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); - }); - arguments.forEach((argName, entityInputs) -> { - aggState.getArguments().put(argName, new RelatedEntitiesArgumentEntry(entityInputs, false)); - }); - aggState.setLastArgsRefreshTs(aggregationStateProto.getLastArgsUpdateTs()); - } } return state; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java index 5d13db2348..31cab71e0e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java @@ -32,7 +32,9 @@ public enum ComponentLifecycleEvent implements Serializable { STOPPED(5), DELETED(6), FAILED(7), - DEACTIVATED(8); + DEACTIVATED(8), + RELATION_UPDATED(9), + RELATION_DELETED(10); @Getter private final int protoNumber; // corresponds to ComponentLifecycleEvent proto diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 2e3a2387e7..bd441dd679 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -916,11 +916,6 @@ message GeofencingArgumentProto { repeated GeofencingZoneProto zones = 2; } -message RelatedEntitiesAggregationStateProto { - int64 lastArgsUpdateTs = 1; - repeated SingleValueArgumentProto aggArguments = 2; -} - message CalculatedFieldStateProto { CalculatedFieldEntityCtxIdProto id = 1; string type = 2; @@ -928,7 +923,7 @@ message CalculatedFieldStateProto { repeated TsRollingArgumentProto rollingValueArguments = 4; repeated GeofencingArgumentProto geofencingArguments = 5; AlarmStateProto alarmState = 6; - RelatedEntitiesAggregationStateProto relatedEntitiesAggregationState = 7; + int64 lastArgsUpdateTs = 7; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. @@ -1281,6 +1276,8 @@ enum ComponentLifecycleEvent { DELETED = 6; FAILED = 7; DEACTIVATED = 8; + RELATION_UPDATED = 9; + RELATION_DELETED = 10; } message ComponentLifecycleMsgProto { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java index 62d6d3d002..2fb12917ff 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java @@ -29,7 +29,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = TbelCfTsRollingArg.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = TbelCfGeofencingArg.class, name = "GEOFENCING_CF_ARGUMENT_VALUE"), @JsonSubTypes.Type(value = TbelCfPropagationArg.class, name = "PROPAGATION_CF_ARGUMENT_VALUE"), - @JsonSubTypes.Type(value = TbelCfRelatedEntitiesAggregation.class, name = "LATEST_VALUES_AGGREGATION") + @JsonSubTypes.Type(value = TbelCfRelatedEntitiesAggregation.class, name = "RELATED_ENTITIES_AGGREGATION") }) public interface TbelCfArg extends TbelCfObject { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java index 75c17f0a0e..3373aa2474 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java @@ -34,7 +34,7 @@ public class TbelCfRelatedEntitiesAggregation implements TbelCfArg { @Override public String getType() { - return "LATEST_VALUES_AGGREGATION"; + return "RELATED_ENTITIES_AGGREGATION"; } @Override From c46d2f041568d6a7bb2e851d2bd00d192744d576 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Oct 2025 10:49:07 +0300 Subject: [PATCH 17/33] added related entities argument entry implementation --- ...titiesAggregationCalculatedFieldState.java | 4 +-- .../RelatedEntitiesArgumentEntry.java | 25 ++++++++++++------- .../function/CountUniqueAggEntry.java | 1 - .../server/utils/CalculatedFieldUtils.java | 2 +- .../RelatedEntitiesArgumentEntryTest.java | 10 ++++---- .../script/api/tbel/TbelCfArg.java | 2 +- ...> TbelCfRelatedEntitiesArgumentValue.java} | 17 +++++++------ 7 files changed, 34 insertions(+), 27 deletions(-) rename common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/{TbelCfRelatedEntitiesAggregation.java => TbelCfRelatedEntitiesArgumentValue.java} (68%) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java index c8731b71f7..7e530b6809 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java @@ -118,7 +118,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat public void cleanupEntityData(EntityId relatedEntityId) { arguments.values().forEach(argEntry -> { RelatedEntitiesArgumentEntry aggEntry = (RelatedEntitiesArgumentEntry) argEntry; - aggEntry.getAggInputs().remove(relatedEntityId); + aggEntry.getEntityInputs().remove(relatedEntityId); }); lastMetricsEvalTs = -1; lastArgsRefreshTs = System.currentTimeMillis(); @@ -135,7 +135,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat for (Map.Entry argEntry : arguments.entrySet()) { String key = argEntry.getKey(); RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry.getValue(); - relatedEntitiesArgumentEntry.getAggInputs().forEach((entityId, argumentEntry) -> { + relatedEntitiesArgumentEntry.getEntityInputs().forEach((entityId, argumentEntry) -> { inputs.computeIfAbsent(entityId, k -> new HashMap<>()).put(key, argumentEntry); }); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java index 5b97b1ed0a..2abe78d243 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java @@ -18,19 +18,21 @@ package org.thingsboard.server.service.cf.ctx.state.aggregation; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.script.api.tbel.TbelCfArg; -import org.thingsboard.script.api.tbel.TbelCfRelatedEntitiesAggregation; +import org.thingsboard.script.api.tbel.TbelCfRelatedEntitiesArgumentValue; +import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import java.util.Map; +import java.util.stream.Collectors; @Data @AllArgsConstructor public class RelatedEntitiesArgumentEntry implements ArgumentEntry { - private final Map aggInputs; + private final Map entityInputs; private boolean forceResetPrevious; @@ -41,24 +43,24 @@ public class RelatedEntitiesArgumentEntry implements ArgumentEntry { @Override public Object getValue() { - return aggInputs; + return entityInputs; } @Override public boolean updateEntry(ArgumentEntry entry) { if (entry instanceof RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry) { - aggInputs.putAll(relatedEntitiesArgumentEntry.aggInputs); + entityInputs.putAll(relatedEntitiesArgumentEntry.entityInputs); return true; } else if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { if (entry.isForceResetPrevious()) { - aggInputs.put(singleValueArgumentEntry.getEntityId(), singleValueArgumentEntry); + entityInputs.put(singleValueArgumentEntry.getEntityId(), singleValueArgumentEntry); return true; } - ArgumentEntry argumentEntry = aggInputs.get(singleValueArgumentEntry.getEntityId()); + ArgumentEntry argumentEntry = entityInputs.get(singleValueArgumentEntry.getEntityId()); if (argumentEntry != null) { argumentEntry.updateEntry(singleValueArgumentEntry); } else { - aggInputs.put(singleValueArgumentEntry.getEntityId(), singleValueArgumentEntry); + entityInputs.put(singleValueArgumentEntry.getEntityId(), singleValueArgumentEntry); } return true; } else { @@ -68,12 +70,17 @@ public class RelatedEntitiesArgumentEntry implements ArgumentEntry { @Override public boolean isEmpty() { - return aggInputs.isEmpty(); + return entityInputs.isEmpty(); } @Override public TbelCfArg toTbelCfArg() { - return new TbelCfRelatedEntitiesAggregation(aggInputs.values()); + var inputs = entityInputs.entrySet().stream() + .collect(Collectors.toMap( + e -> e.getKey().getId(), + e -> (TbelCfSingleValueArg) e.getValue().toTbelCfArg() + )); + return new TbelCfRelatedEntitiesArgumentValue(inputs); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java index efb4a58c90..a66cbaa6af 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.cf.ctx.state.aggregation.function; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; -import java.util.HashSet; import java.util.Optional; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index ba3aa3fd53..fd16245695 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -105,7 +105,7 @@ public class CalculatedFieldUtils { builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); case RELATED_ENTITIES -> { RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry; - relatedEntitiesArgumentEntry.getAggInputs() + relatedEntitiesArgumentEntry.getEntityInputs() .forEach((entityId, entry) -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) entry))); } } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java index 61b45b83c9..cc60b249ac 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java @@ -67,10 +67,10 @@ public class RelatedEntitiesArgumentEntryTest { assertThat(entry.updateEntry(relatedEntitiesArgumentEntry)).isTrue(); - Map aggInputs = entry.getAggInputs(); + Map aggInputs = entry.getEntityInputs(); assertThat(aggInputs.size()).isEqualTo(4); - assertThat(aggInputs.get(device3)).isEqualTo(relatedEntitiesArgumentEntry.getAggInputs().get(device3)); - assertThat(aggInputs.get(device4)).isEqualTo(relatedEntitiesArgumentEntry.getAggInputs().get(device4)); + assertThat(aggInputs.get(device3)).isEqualTo(relatedEntitiesArgumentEntry.getEntityInputs().get(device3)); + assertThat(aggInputs.get(device4)).isEqualTo(relatedEntitiesArgumentEntry.getEntityInputs().get(device4)); } @Test @@ -81,7 +81,7 @@ public class RelatedEntitiesArgumentEntryTest { assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); - Map aggInputs = entry.getAggInputs(); + Map aggInputs = entry.getEntityInputs(); assertThat(aggInputs.size()).isEqualTo(3); assertThat(aggInputs.get(device3)).isEqualTo(singleEntityArgumentEntry); } @@ -92,7 +92,7 @@ public class RelatedEntitiesArgumentEntryTest { assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); - Map aggInputs = entry.getAggInputs(); + Map aggInputs = entry.getEntityInputs(); assertThat(aggInputs.size()).isEqualTo(2); assertThat(aggInputs.get(device2)).isEqualTo(singleEntityArgumentEntry); } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java index 2fb12917ff..4f2719fb75 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java @@ -29,7 +29,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = TbelCfTsRollingArg.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = TbelCfGeofencingArg.class, name = "GEOFENCING_CF_ARGUMENT_VALUE"), @JsonSubTypes.Type(value = TbelCfPropagationArg.class, name = "PROPAGATION_CF_ARGUMENT_VALUE"), - @JsonSubTypes.Type(value = TbelCfRelatedEntitiesAggregation.class, name = "RELATED_ENTITIES_AGGREGATION") + @JsonSubTypes.Type(value = TbelCfRelatedEntitiesArgumentValue.class, name = "RELATED_ENTITIES_ARGUMENT_VALUE") }) public interface TbelCfArg extends TbelCfObject { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesArgumentValue.java similarity index 68% rename from common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java rename to common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesArgumentValue.java index 3373aa2474..02d641d576 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesArgumentValue.java @@ -19,22 +19,23 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; +import java.util.Collections; +import java.util.Map; +import java.util.UUID; + @Data -public class TbelCfRelatedEntitiesAggregation implements TbelCfArg { +public class TbelCfRelatedEntitiesArgumentValue implements TbelCfArg { - private final Object value; + private final Map entityInputs; @JsonCreator - public TbelCfRelatedEntitiesAggregation( - @JsonProperty("value") Object value - ) { - this.value = value; + public TbelCfRelatedEntitiesArgumentValue(@JsonProperty("entityInputs") Map values) { + this.entityInputs = Collections.unmodifiableMap(values); } - @Override public String getType() { - return "RELATED_ENTITIES_AGGREGATION"; + return "RELATED_ENTITIES_ARGUMENT_VALUE"; } @Override From cd15206061d80f9998d95273613ffa3f9126347d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Oct 2025 12:12:09 +0300 Subject: [PATCH 18/33] relation changed processing --- ...alculatedFieldManagerMessageProcessor.java | 57 ++++++++----------- .../queue/DefaultTbClusterService.java | 2 - .../msg/plugin/ComponentLifecycleMsg.java | 6 +- .../server/common/util/ProtoUtils.java | 2 - common/proto/src/main/proto/queue.proto | 1 - 5 files changed, 26 insertions(+), 42 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 3d61825ebe..96f7e9aa80 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; @@ -77,6 +78,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; +import java.util.function.Function; import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; @@ -188,16 +190,12 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) throws CalculatedFieldException { var event = msg.getData().getEvent(); - if (msg.getData().isRelationChanged()) { - log.debug("Processing relation [{}] event: ", msg.getData().getEvent()); - switch (event) { - case RELATION_UPDATED -> onRelationUpdated(msg.getData(), msg.getCallback()); - case RELATION_DELETED -> onRelationDeleted(msg.getData(), msg.getCallback()); - default -> msg.getCallback().onSuccess(); - } + if (ComponentLifecycleEvent.RELATION_UPDATED.equals(event) || ComponentLifecycleEvent.RELATION_DELETED.equals(event)) { + log.debug("Processing relation [{}] event from entity: [{}]", event, msg.getData().getEntityId()); + onRelationChangedEvent(msg.getData(), msg.getCallback()); return; } - log.debug("Processing entity lifecycle event: [{}] for entity: [{}]", msg.getData().getEvent(), msg.getData().getEntityId()); + log.debug("Processing entity lifecycle event: [{}] for entity: [{}]", event, msg.getData().getEntityId()); var entityType = msg.getData().getEntityId().getEntityType(); switch (entityType) { case CALCULATED_FIELD -> { @@ -306,36 +304,29 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } } - private void onRelationUpdated(ComponentLifecycleMsg msg, TbCallback callback) { - try { - EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); - EntityId toId = entityRelation.getTo(); - EntityId fromId = entityRelation.getFrom(); - String relationType = entityRelation.getType(); - - MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); - processRelationByDirection(EntitySearchDirection.TO, relationType, toId, callbackForToAndFrom, (entityId, ctx, cb) -> initRelatedEntity(entityId, fromId, ctx, cb)); - processRelationByDirection(EntitySearchDirection.FROM, relationType, fromId, callbackForToAndFrom, (entityId, ctx, cb) -> initRelatedEntity(entityId, toId, ctx, cb)); - } catch (Exception e) { - callback.onSuccess(); - } - } + private void onRelationChangedEvent(ComponentLifecycleMsg msg, TbCallback callback) { + Function> relationAction = switch (msg.getEvent()) { + case RELATION_UPDATED -> relatedId -> (entityId, ctx, cb) -> initRelatedEntity(entityId, relatedId, ctx, cb); + case RELATION_DELETED -> relatedId -> (entityId, ctx, cb) -> deleteRelatedEntity(entityId, relatedId, ctx, cb); + default -> null; + }; - private void onRelationDeleted(ComponentLifecycleMsg msg, TbCallback callback) { - try { - EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); - EntityId toId = entityRelation.getTo(); - EntityId fromId = entityRelation.getFrom(); - String relationType = entityRelation.getType(); - - MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); - processRelationByDirection(EntitySearchDirection.TO, relationType, toId, callbackForToAndFrom, (entityId, ctx, cb) -> deleteRelatedEntity(entityId, fromId, ctx, cb)); - processRelationByDirection(EntitySearchDirection.FROM, relationType, fromId, callbackForToAndFrom, (entityId, ctx, cb) -> deleteRelatedEntity(entityId, toId, ctx, cb)); - } catch (Exception e) { + if (relationAction == null) { callback.onSuccess(); + return; } + + EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); + EntityId toId = entityRelation.getTo(); + EntityId fromId = entityRelation.getFrom(); + String relationType = entityRelation.getType(); + + MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); + processRelationByDirection(EntitySearchDirection.TO, relationType, toId, callbackForToAndFrom, relationAction.apply(fromId)); + processRelationByDirection(EntitySearchDirection.FROM, relationType, fromId, callbackForToAndFrom, relationAction.apply(toId)); } + private void processRelationByDirection(EntitySearchDirection direction, String relationType, EntityId mainId, diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 5b6596c9c6..dde24d358a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -740,7 +740,6 @@ public class DefaultTbClusterService implements TbClusterService { ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() .tenantId(tenantId) .entityId(entityRelation.getFrom()) - .relationChanged(true) .event(ComponentLifecycleEvent.RELATION_UPDATED) .info(JacksonUtil.valueToTree(entityRelation)) .build(); @@ -752,7 +751,6 @@ public class DefaultTbClusterService implements TbClusterService { ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() .tenantId(tenantId) .entityId(entityRelation.getFrom()) - .relationChanged(true) .event(ComponentLifecycleEvent.RELATION_DELETED) .info(JacksonUtil.valueToTree(entityRelation)) .build(); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index 38df529c96..23b9fe08e3 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -47,15 +47,14 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { private final EntityId oldProfileId; private final EntityId profileId; private final boolean ownerChanged; - private final boolean relationChanged; private final JsonNode info; public ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event) { - this(tenantId, entityId, event, null, null, null, null, false, false, null); + this(tenantId, entityId, event, null, null, null, null, false, null); } @Builder - private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId, boolean ownerChanged, boolean relationChanged, JsonNode info) { + private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId, boolean ownerChanged, JsonNode info) { this.tenantId = tenantId; this.entityId = entityId; this.event = event; @@ -64,7 +63,6 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { this.oldProfileId = oldProfileId; this.profileId = profileId; this.ownerChanged = ownerChanged; - this.relationChanged = relationChanged; this.info = info; } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 83fb158efd..26a64c7f8a 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -130,7 +130,6 @@ public class ProtoUtils { builder.setOldProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); } builder.setOwnerChanged(msg.isOwnerChanged()); - builder.setRelationChanged(msg.isRelationChanged()); if (msg.getName() != null) { builder.setName(msg.getName()); } @@ -168,7 +167,6 @@ public class ProtoUtils { builder.oldProfileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB()))); } builder.ownerChanged(proto.getOwnerChanged()); - builder.relationChanged(proto.getRelationChanged()); if (proto.hasInfo()) { builder.info(JacksonUtil.toJsonNode(proto.getInfo())); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index bd441dd679..b060b47429 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1296,7 +1296,6 @@ message ComponentLifecycleMsgProto { int64 profileIdLSB = 12; optional string info = 13; bool ownerChanged = 100; - bool relationChanged = 14; } message EdgeEventMsgProto { From f48b8752d225e31f85b2acd745c8d5babc21bc0d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Oct 2025 14:33:12 +0300 Subject: [PATCH 19/33] fixed agg cfs filtration in queueu service --- .../service/cf/DefaultCalculatedFieldQueueService.java | 9 ++++++++- .../service/cf/ctx/state/CalculatedFieldState.java | 2 -- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 84f49c7c9e..c7e369a862 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -199,7 +199,14 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS RelationPathLevel inverseRelation = new RelationPathLevel(inverseDirection, relation.relationType()); List byRelationPathQuery = relationService.findByRelationPathQuery(tenantId, new EntityRelationPathQuery(entityId, List.of(inverseRelation))); if (!byRelationPathQuery.isEmpty()) { - return true; + EntityId cfEntityId = cfCtx.getEntityId(); + for (EntityRelation entityRelation : byRelationPathQuery) { + EntityId relatedId = (inverseDirection == EntitySearchDirection.FROM) ? entityRelation.getTo() : entityRelation.getFrom(); + if (cfEntityId.equals(relatedId) || cfEntityId.equals(calculatedFieldCache.getProfileId(tenantId, relatedId))) { + return true; + } + } + return false; } } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 2b3ba19528..34e9dad439 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -55,8 +55,6 @@ public interface CalculatedFieldState extends Closeable { long getLatestTimestamp(); - CalculatedFieldCtx getCtx(); - void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx); void init(); From 269794ec27d9faa2452c3761e11908968f25f95b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Oct 2025 15:54:33 +0300 Subject: [PATCH 20/33] added json subtype --- .../server/service/cf/ctx/state/CalculatedFieldState.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 34e9dad439..9598cc2b49 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -42,7 +43,8 @@ import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArg @Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), @Type(value = GeofencingCalculatedFieldState.class, name = "GEOFENCING"), @Type(value = AlarmCalculatedFieldState.class, name = "ALARM"), - @Type(value = PropagationCalculatedFieldState.class, name = "PROPAGATION") + @Type(value = PropagationCalculatedFieldState.class, name = "PROPAGATION"), + @Type(value = RelatedEntitiesAggregationCalculatedFieldState.class, name = "RELATED_ENTITIES_AGGREGATION") }) public interface CalculatedFieldState extends Closeable { From 6dba0b6fd21ce4a6d1a388a88a60915f2f26e10f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 23 Oct 2025 11:19:53 +0300 Subject: [PATCH 21/33] scheduling for agg cfs on restart --- .../CalculatedFieldManagerMessageProcessor.java | 5 ++++- .../RelatedEntitiesAggregationCalculatedFieldState.java | 6 ------ ...tedEntitiesAggregationCalculatedFieldConfiguration.java | 7 +++++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 40707e32f3..de3967d5b6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -652,7 +652,10 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private List getCalculatedFieldsByEntityIdAndProfile(EntityId entityId) { List cfsByEntityIdAndProfile = new ArrayList<>(); cfsByEntityIdAndProfile.addAll(getCalculatedFieldsByEntityId(entityId)); - cfsByEntityIdAndProfile.addAll(getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId))); + EntityId profileId = getProfileId(tenantId, entityId); + if (profileId != null) { + cfsByEntityIdAndProfile.addAll(getCalculatedFieldsByEntityId(profileId)); + } return cfsByEntityIdAndProfile; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java index 7e530b6809..655217263b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java @@ -75,12 +75,6 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat metrics = null; } - @Override - public void init() { - super.init(); - ctx.scheduleReevaluation(deduplicationIntervalMs, actorCtx); - } - @Override public CalculatedFieldType getType() { return CalculatedFieldType.RELATED_ENTITIES_AGGREGATION; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java index 9d4c7bdaf6..931cb919ec 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation; +import com.fasterxml.jackson.annotation.JsonIgnore; import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; @@ -56,4 +57,10 @@ public class RelatedEntitiesAggregationCalculatedFieldConfiguration implements A } } + @JsonIgnore + @Override + public boolean requiresScheduledReevaluation() { + return true; + } + } From bb308919ce3d8ad297b79c4288bf93c8f3230289 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 23 Oct 2025 15:43:43 +0300 Subject: [PATCH 22/33] UI: Add new tenant profile configuration minAllowedDeduplicationIntervalInSecForCF --- ui-ngx/src/app/core/auth/auth.models.ts | 1 + ui-ngx/src/app/core/auth/auth.reducer.ts | 1 + ...ult-tenant-profile-configuration.component.html | 14 +++++++++++++- ...fault-tenant-profile-configuration.component.ts | 1 + ui-ngx/src/app/shared/models/tenant.model.ts | 2 ++ .../src/assets/locale/locale.constant-en_US.json | 3 +++ 6 files changed, 21 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/core/auth/auth.models.ts b/ui-ngx/src/app/core/auth/auth.models.ts index d6612f2427..6e4d324b5b 100644 --- a/ui-ngx/src/app/core/auth/auth.models.ts +++ b/ui-ngx/src/app/core/auth/auth.models.ts @@ -31,6 +31,7 @@ export interface SysParamsState { maxDebugModeDurationMinutes: number; maxDataPointsPerRollingArg: number; maxArgumentsPerCF: number; + minAllowedDeduplicationIntervalInSecForCF: number; minAllowedScheduledUpdateIntervalInSecForCF: number; maxRelationLevelPerCfArgument: number; ruleChainDebugPerTenantLimitsConfiguration?: string; diff --git a/ui-ngx/src/app/core/auth/auth.reducer.ts b/ui-ngx/src/app/core/auth/auth.reducer.ts index 8cfbc04197..777cf5308e 100644 --- a/ui-ngx/src/app/core/auth/auth.reducer.ts +++ b/ui-ngx/src/app/core/auth/auth.reducer.ts @@ -33,6 +33,7 @@ const emptyUserAuthState: AuthPayload = { mobileQrEnabled: false, maxResourceSize: 0, maxArgumentsPerCF: 0, + minAllowedDeduplicationIntervalInSecForCF: 0, minAllowedScheduledUpdateIntervalInSecForCF: 0, maxRelationLevelPerCfArgument: 0, maxDataPointsPerRollingArg: 0, diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html index d53cacbd83..8cfa8fd0ed 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -354,7 +354,19 @@ tenant-profile.relation-search-entity-limit-hint -
+ + tenant-profile.min-allowed-deduplication-interval + + + {{ 'tenant-profile.min-allowed-deduplication-interval-required' | translate}} + + + {{ 'tenant-profile.min-allowed-deduplication-interval-range' | translate}} + + + diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts index 9595def95e..0000d01995 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts @@ -116,6 +116,7 @@ export class DefaultTenantProfileConfigurationComponent implements ControlValueA maxCalculatedFieldsPerEntity: [0, [Validators.required, Validators.min(0)]], maxArgumentsPerCF: [0, [Validators.required, Validators.min(0)]], maxRelationLevelPerCfArgument: [1, [Validators.required, Validators.min(1)]], + minAllowedDeduplicationIntervalInSecForCF: [0, [Validators.required, Validators.min(0)]], maxRelatedEntitiesToReturnPerCfArgument: [1, [Validators.required, Validators.min(1)]], minAllowedScheduledUpdateIntervalInSecForCF: [0, [Validators.required, Validators.min(0)]], maxDataPointsPerRollingArg: [0, [Validators.required, Validators.min(0)]], diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 1ed1092207..ae7a0ae8b8 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -107,6 +107,7 @@ export interface DefaultTenantProfileConfiguration { maxCalculatedFieldsPerEntity: number; maxArgumentsPerCF: number; maxRelationLevelPerCfArgument: number; + minAllowedDeduplicationIntervalInSecForCF: number; maxRelatedEntitiesToReturnPerCfArgument: number; minAllowedScheduledUpdateIntervalInSecForCF: number; maxDataPointsPerRollingArg: number; @@ -174,6 +175,7 @@ export function createTenantProfileConfiguration(type: TenantProfileType): Tenan maxArgumentsPerCF: 10, maxDataPointsPerRollingArg: 1000, maxRelationLevelPerCfArgument: 10, + minAllowedDeduplicationIntervalInSecForCF: 3600, maxRelatedEntitiesToReturnPerCfArgument: 100, minAllowedScheduledUpdateIntervalInSecForCF: 0, maxStateSizeInKBytes: 32, diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 3ee738a8b0..331f3cdb09 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -5876,6 +5876,9 @@ "max-related-level-per-argument-required": "Relation level per 'Related entities' argument max number is required", "min-allowed-scheduled-update-interval": "Min allowed update interval for 'Related entities' arguments (seconds)", "min-allowed-scheduled-update-interval-range": "Min allowed update interval min number can't be negative", + "min-allowed-deduplication-interval": "Min allowed deduplication interval (seconds)", + "min-allowed-deduplication-interval-range": "Min allowed deduplication interval value can't be negative", + "min-allowed-deduplication-interval-required": "Min allowed deduplication interval is required", "min-allowed-scheduled-update-interval-required": "Min allowed update interval min number is required", "max-state-size": "State maximum size in KB", "max-state-size-range": "State maximum size in KB can't be negative", From 59d2fe8c7b40866cd97f58cff2162d9a2aa0e905 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 23 Oct 2025 19:06:55 +0300 Subject: [PATCH 23/33] UI: Add bacis config RELATED_ENTITIES_AGGREGATION cf --- .../calculated-field.module.ts | 4 + ...ulated-field-argument-panel.component.html | 54 +++--- ...lculated-field-argument-panel.component.ts | 11 ++ ...calculated-field-arguments-table.module.ts | 9 +- ...d-aggregation-arguments-table.component.ts | 70 ++++++++ .../calculated-field-dialog.component.html | 7 + .../calculated-field-dialog.component.scss | 3 + .../calculated-field-output.component.html | 85 +++++----- .../calculated-field-output.component.ts | 13 +- ...ities-aggregation-component.component.html | 75 +++++++++ ...ntities-aggregation-component.component.ts | 154 ++++++++++++++++++ ...d-entities-aggregation-component.module.ts | 45 +++++ .../components/time-unit-input.component.html | 6 +- .../shared/models/calculated-field.models.ts | 24 ++- .../assets/locale/locale.constant-en_US.json | 15 +- 15 files changed, 505 insertions(+), 70 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/related-aggregation-arguments-table.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts index e0db7a6eef..5e3a854aa2 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts @@ -38,6 +38,9 @@ import { import { PropagationConfigurationModule } from '@home/components/calculated-fields/components/propagation-configuration/propagation-configuration.module'; +import { + RelatedEntitiesAggregationComponentModule +} from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module'; @NgModule({ declarations: [ @@ -52,6 +55,7 @@ import { EntityDebugSettingsButtonComponent, SimpleConfigurationModule, PropagationConfigurationModule, + RelatedEntitiesAggregationComponentModule, ], exports: [ CalculatedFieldDialogComponent, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html index 77bdedb068..607b107094 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html @@ -18,6 +18,11 @@
{{ 'calculated-fields.argument-settings' | translate }}
+ @if (hint) { +
+ {{ hint | translate }} +
+ }
@if (!isOutputKey) { } -
-
{{ 'entity.entity-type' | translate }}
- - - @for (type of argumentEntityTypes; track type) { - {{ ArgumentEntityTypeTranslations.get(type) | translate }} + @if (!hiddenEntityTypes) { +
+
{{ 'entity.entity-type' | translate }}
+ + + @for (type of argumentEntityTypes; track type) { + {{ ArgumentEntityTypeTranslations.get(type) | translate }} + } + + @if (argumentType.touched && argumentType.hasError('required')) { + + warning + } - - @if (argumentType.touched && argumentType.hasError('required')) { - - warning - - } - -
+ +
+ } @if (ArgumentEntityTypeParamsMap.has(entityType)) {
{{ ArgumentEntityTypeParamsMap.get(entityType).title | translate }}
@@ -143,9 +150,18 @@ } @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Rolling) {
-
{{ 'calculated-fields.default-value' | translate }}
+
{{ 'calculated-fields.default-value' | translate }}
+ @if (argumentFormGroup.get('defaultValue').touched && argumentFormGroup.get('defaultValue').hasError('required')) { + + warning + + }
} @else { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts index 070ffb5c06..6f90d126e0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts @@ -52,6 +52,7 @@ import { Store } from '@ngrx/store'; import { EntityAutocompleteComponent } from '@shared/components/entity/entity-autocomplete.component'; import { NULL_UUID } from '@shared/models/id/has-uuid'; import { TenantId } from '@shared/models/id/tenant-id'; +import { deduplicationStrategiesHintTranslations } from '@home/components/rule-node/rule-node-config.models'; @Component({ selector: 'tb-calculated-field-argument-panel', @@ -68,6 +69,9 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI @Input() isScript: boolean; @Input() usedArgumentNames: string[]; @Input() isOutputKey = false; + @Input() hiddenEntityTypes = false; + @Input() defaultValueRequired = false; + @Input() hint: string; @Input() argumentEntityTypes = Object.values(ArgumentEntityType).filter(value => value !== ArgumentEntityType.RelationQuery) as ArgumentEntityType[]; @ViewChild('entityAutocomplete') entityAutocomplete: EntityAutocompleteComponent; @@ -146,6 +150,11 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI this.setInitialEntityType(); this.setWatchKeyChange(); + if (this.defaultValueRequired) { + this.argumentFormGroup.get('defaultValue').addValidators(Validators.required); + this.argumentFormGroup.get('defaultValue').updateValueAndValidity({onlySelf: true}); + } + this.argumentTypes = Object.values(ArgumentType) .filter(type => type !== ArgumentType.Rolling || this.isScript); } @@ -311,4 +320,6 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI this.entityNameSubject.next(null); } } + + protected readonly deduplicationStrategiesHintTranslations = deduplicationStrategiesHintTranslations; } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts index 082001f052..cae0b92387 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts @@ -26,6 +26,9 @@ import { import { PropagateArgumentsTableComponent } from '@home/components/calculated-fields/components/calculated-field-arguments/propagate-arguments-table.component'; +import { + RelatedAggregationArgumentsTableComponent +} from '@home/components/calculated-fields/components/calculated-field-arguments/related-aggregation-arguments-table.component'; @NgModule({ imports: [ @@ -35,11 +38,13 @@ import { declarations: [ CalculatedFieldArgumentPanelComponent, CalculatedFieldArgumentsTableComponent, - PropagateArgumentsTableComponent + PropagateArgumentsTableComponent, + RelatedAggregationArgumentsTableComponent ], exports: [ CalculatedFieldArgumentsTableComponent, - PropagateArgumentsTableComponent + PropagateArgumentsTableComponent, + RelatedAggregationArgumentsTableComponent ] }) export class CalculatedFieldArgumentsTableModule {} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/related-aggregation-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/related-aggregation-arguments-table.component.ts new file mode 100644 index 0000000000..7c9212d3c2 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/related-aggregation-arguments-table.component.ts @@ -0,0 +1,70 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { ChangeDetectorRef, Component, DestroyRef, forwardRef, Renderer2, ViewContainerRef, } from '@angular/core'; +import { FormBuilder, NG_VALIDATORS, NG_VALUE_ACCESSOR, } from '@angular/forms'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityService } from '@core/http/entity.service'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { + CalculatedFieldArgumentsTableComponent +} from '@home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.component'; +import { ArgumentEntityType } from '@shared/models/calculated-field.models'; + +@Component({ + selector: 'tb-related-aggregation-arguments-table', + templateUrl: './calculated-field-arguments-table.component.html', + styleUrls: [`calculated-field-arguments-table.component.scss`], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => RelatedAggregationArgumentsTableComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => RelatedAggregationArgumentsTableComponent), + multi: true + } + ], +}) +export class RelatedAggregationArgumentsTableComponent extends CalculatedFieldArgumentsTableComponent { + + constructor( + protected fb: FormBuilder, + protected popoverService: TbPopoverService, + protected viewContainerRef: ViewContainerRef, + protected cd: ChangeDetectorRef, + protected renderer: Renderer2, + protected entityService: EntityService, + protected destroyRef: DestroyRef, + protected store: Store + ) { + super(fb, popoverService, viewContainerRef, cd, renderer, entityService, destroyRef, store); + + this.argumentNameColumn = 'calculated-fields.argument-name'; + this.displayColumns = ['name', 'type', 'key', 'actions']; + this.panelAdditionalCtx = { + hiddenEntityTypes: true, + defaultValueRequired: true, + argumentEntityTypes: [ArgumentEntityType.Current], + hint: 'calculated-fields.hint.setting-arguments-aggregation' + }; + + this.isScript = false; + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 1d9dcc98f1..478d41f42b 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -75,6 +75,13 @@ [testScript]="onTestScript.bind(this)"> } + @case (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION) { + + + } @default { @if (simpleMode) { -
- - - {{ - (outputForm.get('type').value === OutputType.Timeseries - ? 'calculated-fields.timeseries-key' - : 'calculated-fields.attribute-key') - | translate - }} - - - @if (outputForm.get('name').errors && outputForm.get('name').touched) { - - @if (outputForm.get('name').hasError('required')) { - {{ 'common.hint.key-required' | translate }} - } @else if (outputForm.get('name').hasError('pattern')) { - {{ 'common.hint.key-pattern' | translate }} - } @else if (outputForm.get('name').hasError('maxlength')) { - {{ 'common.hint.key-max-length' | translate }} - } - - } - - - {{ 'calculated-fields.decimals-by-default' | translate }} - - @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { - {{ 'calculated-fields.hint.decimals-range' | translate }} - } - -
- - - - - - - - - + @if (hiddenName) { +
+ + {{ 'calculated-fields.decimals-by-default' | translate }} + + @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { + {{ 'calculated-fields.hint.decimals-range' | translate }} + } + + +
+ } @else { +
+ + + {{ + (outputForm.get('type').value === OutputType.Timeseries + ? 'calculated-fields.timeseries-key' + : 'calculated-fields.attribute-key') + | translate + }} + + + @if (outputForm.get('name').errors && outputForm.get('name').touched) { + + @if (outputForm.get('name').hasError('required')) { + {{ 'common.hint.key-required' | translate }} + } @else if (outputForm.get('name').hasError('pattern')) { + {{ 'common.hint.key-pattern' | translate }} + } @else if (outputForm.get('name').hasError('maxlength')) { + {{ 'common.hint.key-max-length' | translate }} + } + + } + + + {{ 'calculated-fields.decimals-by-default' | translate }} + + @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { + {{ 'calculated-fields.hint.decimals-range' | translate }} + } + +
+ + } }
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts index 9a95c921fa..464ca99bd2 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts @@ -35,6 +35,7 @@ import { digitsRegex, oneSpaceInsideRegex } from '@shared/models/regex.constants import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityType } from '@shared/models/entity-type.models'; +import { coerceBoolean } from '@shared/decorators/coercion'; @Component({ selector: 'tb-calculate-field-output', @@ -55,8 +56,13 @@ import { EntityType } from '@shared/models/entity-type.models'; export class CalculatedFieldOutputComponent implements ControlValueAccessor, Validator, OnInit, OnChanges { @Input() + @coerceBoolean() simpleMode = false; + @Input() + @coerceBoolean() + hiddenName = false; + @Input({required: true}) entityId: EntityId; @@ -137,11 +143,14 @@ export class CalculatedFieldOutputComponent implements ControlValueAccessor, Val } private updatedFormWithMode(): void { - if (this.simpleMode) { + if (this.simpleMode && !this.hiddenName) { this.outputForm.get('name').enable({emitEvent: false}); - this.outputForm.get('decimalsByDefault').enable({emitEvent: false}); } else { this.outputForm.get('name').disable({emitEvent: false}); + } + if (this.simpleMode) { + this.outputForm.get('decimalsByDefault').enable({emitEvent: false}); + } else { this.outputForm.get('decimalsByDefault').disable({emitEvent: false}); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html new file mode 100644 index 0000000000..1988141c2a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html @@ -0,0 +1,75 @@ + +
+
+
+ {{ 'calculated-fields.aggregation-path-related-entities' | translate }} +
+
+ + {{ 'calculated-fields.direction' | translate }} + + @for (direction of Directions; track direction) { + {{ PropagationDirectionTranslations.get(direction) | translate }} + } + + + + +
+
+
+
+ {{ 'calculated-fields.arguments' | translate }} +
+ +
+
+
+ {{ 'calculated-fields.metrics' | translate }} +
+ + +
+ +
+ +
+ calculated-fields.use-latest-timestamp +
+ +
+ +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.ts new file mode 100644 index 0000000000..d10372beea --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.ts @@ -0,0 +1,154 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { EntityId } from '@shared/models/id/entity-id'; +import { Observable, of } from 'rxjs'; +import { + CalculatedFieldOutput, + CalculatedFieldRelatedAggregationConfiguration, + CalculatedFieldType, + getCalculatedFieldArgumentsEditorCompleter, + getCalculatedFieldArgumentsHighlights, + OutputType, + PropagationDirectionTranslations +} from '@shared/models/calculated-field.models'; +import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; +import { map } from 'rxjs/operators'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { ScriptLanguage } from '@app/shared/models/rule-node.models'; +import { EntitySearchDirection } from '@shared/models/relation.models'; +import { getCurrentAuthState } from '@core/auth/auth.selectors'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; + +@Component({ + selector: 'tb-related-entities-aggregation-component', + templateUrl: './related-entities-aggregation-component.component.html', + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => RelatedEntitiesAggregationComponentComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => RelatedEntitiesAggregationComponentComponent), + multi: true + } + ], +}) +export class RelatedEntitiesAggregationComponentComponent implements ControlValueAccessor, Validator { + + @Input({required: true}) + entityId: EntityId; + + @Input({required: true}) + tenantId: string; + + @Input({required: true}) + entityName: string; + + relatedAggregationConfiguration = this.fb.group({ + relation: this.fb.group({ + direction: [EntitySearchDirection.FROM, Validators.required], + relationType: ['Contains', Validators.required], + }), + arguments: this.fb.control({}), + deduplicationIntervalInSec: [], + output: this.fb.control({ + scope: AttributeScope.SERVER_SCOPE, + type: OutputType.Timeseries, + }), + useLatestTs: [false] + }); + + readonly ScriptLanguage = ScriptLanguage; + readonly CalculatedFieldType = CalculatedFieldType; + readonly OutputType = OutputType; + readonly Directions = Object.values(EntitySearchDirection) as Array; + readonly PropagationDirectionTranslations = PropagationDirectionTranslations; + readonly minAllowedDeduplicationIntervalInSecForCF = getCurrentAuthState(this.store).minAllowedDeduplicationIntervalInSecForCF; + + + functionArgs$ = this.relatedAggregationConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => ['ctx', ...Object.keys(argumentsObj)]) + ); + + argumentsEditorCompleter$ = this.relatedAggregationConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => getCalculatedFieldArgumentsEditorCompleter(argumentsObj ?? {})) + ); + + argumentsHighlightRules$ = this.relatedAggregationConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => getCalculatedFieldArgumentsHighlights(argumentsObj)) + ); + + private propagateChange: (config: CalculatedFieldRelatedAggregationConfiguration) => void = () => { }; + + constructor(private fb: FormBuilder, + private store: Store) { + + this.relatedAggregationConfiguration.valueChanges.pipe( + takeUntilDestroyed() + ).subscribe((value: CalculatedFieldRelatedAggregationConfiguration) => { + this.updatedModel(value); + }) + } + + validate(): ValidationErrors | null { + return this.relatedAggregationConfiguration.valid || this.relatedAggregationConfiguration.status === "DISABLED" ? null : {invalidPropagateConfig: false}; + } + + writeValue(value: CalculatedFieldRelatedAggregationConfiguration): void { + this.relatedAggregationConfiguration.patchValue(value, {emitEvent: false}); + setTimeout(() => { + this.relatedAggregationConfiguration.get('arguments').updateValueAndValidity({onlySelf: true}); + }); + } + + registerOnChange(fn: (config: CalculatedFieldRelatedAggregationConfiguration) => void): void { + this.propagateChange = fn; + } + + registerOnTouched(_: any): void { } + + setDisabledState(isDisabled: boolean): void { + if (isDisabled) { + this.relatedAggregationConfiguration.disable({emitEvent: false}); + } else { + this.relatedAggregationConfiguration.enable({emitEvent: false}); + } + } + + fetchOptions(searchText: string): Observable> { + const search = searchText ? searchText?.toLowerCase() : ''; + return of(['Contains', 'Manages']).pipe(map(name => name?.filter(option => option.toLowerCase().includes(search)))); + } + + private updatedModel(value: CalculatedFieldRelatedAggregationConfiguration): void { + value.type = CalculatedFieldType.RELATED_ENTITIES_AGGREGATION; + this.propagateChange(value); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts new file mode 100644 index 0000000000..a2c48c1896 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts @@ -0,0 +1,45 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { SharedModule } from '@shared/shared.module'; +import { + CalculatedFieldOutputModule +} from '@home/components/calculated-fields/components/output/calculated-field-output.module'; +import { + CalculatedFieldArgumentsTableModule +} from '@home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module'; +import { + RelatedEntitiesAggregationComponentComponent +} from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component'; + +@NgModule({ + imports: [ + CommonModule, + SharedModule, + CalculatedFieldOutputModule, + CalculatedFieldArgumentsTableModule, + ], + declarations: [ + RelatedEntitiesAggregationComponentComponent, + ], + exports: [ + RelatedEntitiesAggregationComponentComponent, + ] +}) +export class RelatedEntitiesAggregationComponentModule { +} diff --git a/ui-ngx/src/app/shared/components/time-unit-input.component.html b/ui-ngx/src/app/shared/components/time-unit-input.component.html index d5f6319576..53a444895b 100644 --- a/ui-ngx/src/app/shared/components/time-unit-input.component.html +++ b/ui-ngx/src/app/shared/components/time-unit-input.component.html @@ -18,7 +18,7 @@
+ subscriptSizing="dynamic"> @if (labelText && !inlineField) { {{ labelText }} } @@ -41,7 +41,9 @@ {{ hasError }} - @if (!inlineField) { diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 8294a776df..8ea6659bca 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -65,7 +65,8 @@ export enum CalculatedFieldType { SIMPLE = 'SIMPLE', SCRIPT = 'SCRIPT', GEOFENCING = 'GEOFENCING', - PROPAGATION = 'PROPAGATION' + PROPAGATION = 'PROPAGATION', + RELATED_ENTITIES_AGGREGATION = 'RELATED_ENTITIES_AGGREGATION' } export const CalculatedFieldTypeTranslations = new Map( @@ -74,6 +75,7 @@ export const CalculatedFieldTypeTranslations = new Map; + useLatestTs: boolean; output: CalculatedFieldSimpleOutput; } @@ -105,6 +109,15 @@ export interface CalculatedFieldGeofencingConfiguration { output: CalculatedFieldOutput; } +export interface CalculatedFieldRelatedAggregationConfiguration { + type: CalculatedFieldType.RELATED_ENTITIES_AGGREGATION; + relation: RelationPathLevel; + arguments: Record; + deduplicationIntervalInSec: number; + useLatestTs: boolean; + output: Omit; +} + interface BasePropagationConfiguration { type: CalculatedFieldType.PROPAGATION; direction: EntitySearchDirection; @@ -250,7 +263,7 @@ export interface CalculatedFieldGeofencing { export interface RefDynamicSourceConfiguration { type?: ArgumentEntityType.RelationQuery; - levels?: Array<{direction: EntitySearchDirection; relationType: string;}>; + levels?: Array; } export interface CalculatedFieldGeofencingValue extends CalculatedFieldGeofencing { @@ -317,6 +330,11 @@ export interface CalculatedFieldArgumentValueBase { type: ArgumentType; } +export interface RelationPathLevel { + direction: EntitySearchDirection; + relationType: string; +} + export interface CalculatedFieldAttributeArgumentValue extends CalculatedFieldArgumentValueBase { ts: number; value: ValueType; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 331f3cdb09..0333c5ed2e 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1054,7 +1054,8 @@ "simple": "Simple", "script": "Script", "geofencing" : "Geofencing", - "propagation": "Propagation" + "propagation": "Propagation", + "related-entities-aggregation": "Related entities aggregation" }, "arguments": "Arguments", "decimals-by-default": "Decimals by default", @@ -1088,6 +1089,7 @@ "shared-attributes": "Shared attributes", "attribute-key": "Attribute key", "default-value": "Default value", + "default-value-required": "Default value is required.", "limit": "Max values", "time-window": "Time window", "customer-name": "Customer name", @@ -1156,6 +1158,11 @@ "data-propagate": "Data to propagate", "output-key": "Output key", "copy-output-key": "Copy output key", + "aggregation-path-related-entities": "Aggregation path to related entities", + "metrics": "Metrics", + "deduplication-interval": "Deduplication interval", + "deduplication-interval-min": "Deduplication interval should be at least {{ sec }} second.", + "deduplication-interval-required": "Deduplication interval is required.", "hint": { "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with time series rolling type.", "arguments-propagate-arguments-with-rolling": "'Time series rolling' type is incompatible with 'Arguments only' propagation.", @@ -1202,7 +1209,11 @@ "zone-group-refresh-interval-required": "Zone groups refresh interval is required.", "zone-group-refresh-interval-min": "Zone group refresh interval should be at least {{ min }} second.", "propagation-path-related-entities": "Defines a direct, single-level path to a related entity based on the selected direction and relation type.", - "data-propagate": "Defines the data to be propagated from the arguments configured below. 'Arguments only' uses the retrieved data directly, while 'Expression result' calculates a new value from that data." + "data-propagate": "Defines the data to be propagated from the arguments configured below. 'Arguments only' uses the retrieved data directly, while 'Expression result' calculates a new value from that data.", + "aggregation-path-related-entities": "Defines a single-level aggregation path via direct relations with parent or child entities based on direction and relation type. Only relations between device, asset, customer, and tenant entities are supported.", + "arguments-aggregation": "Defines input parameters used for filtering and aggregation.", + "setting-arguments-aggregation": "Data will be fetched from related entities configured in aggregation path.", + "metrics": "Defines metrics aggregated based on the configured arguments." } }, "ai-models": { From aab06c465d32a462fd340e349c42f8afbb5ce6ea Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 24 Oct 2025 16:48:34 +0300 Subject: [PATCH 24/33] UI: Add metrics config to RELATED_ENTITIES_AGGREGATION --- .../calculated-field-output.component.html | 25 +- ...culated-field-metrics-panel.component.html | 168 ++++++++++++ ...culated-field-metrics-panel.component.scss | 31 +++ ...alculated-field-metrics-panel.component.ts | 176 +++++++++++++ ...culated-field-metrics-table.component.html | 111 ++++++++ ...culated-field-metrics-table.component.scss | 76 ++++++ ...alculated-field-metrics-table.component.ts | 242 ++++++++++++++++++ ...ities-aggregation-component.component.html | 7 +- ...ntities-aggregation-component.component.ts | 5 +- ...d-entities-aggregation-component.module.ts | 8 + .../shared/models/calculated-field.models.ts | 49 ++++ .../assets/locale/locale.constant-en_US.json | 30 ++- 12 files changed, 909 insertions(+), 19 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.ts diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html index fcc000bf98..1faeb6adf6 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html @@ -44,13 +44,7 @@ @if (simpleMode) { @if (hiddenName) {
- - {{ 'calculated-fields.decimals-by-default' | translate }} - - @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { - {{ 'calculated-fields.hint.decimals-range' | translate }} - } - +
} @else { @@ -77,15 +71,18 @@ } - - {{ 'calculated-fields.decimals-by-default' | translate }} - - @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { - {{ 'calculated-fields.hint.decimals-range' | translate }} - } - +
} }
+ + + {{ 'calculated-fields.decimals-by-default' | translate }} + + @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { + {{ 'calculated-fields.hint.decimals-range' | translate }} + } + + diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html new file mode 100644 index 0000000000..5c2985321a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html @@ -0,0 +1,168 @@ + +
+
+
{{ 'calculated-fields.metrics.metric-settings' | translate }}
+
+
+
{{ 'calculated-fields.metrics.metric-name' | translate }}
+ + + @if (metricForm.get('name').touched && metricForm.get('name').hasError('required')) { + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('duplicateName')) { + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('pattern')) { + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('maxlength')) { + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('forbiddenName')) { + + warning + + } + +
+
+
{{ 'calculated-fields.metrics.aggregation' | translate }}
+ + + @for (aggFunction of AggFunctions; track aggFunction) { + {{ AggFunctionTranslations.get(aggFunction) | translate }} + } + + +
+ +
+ + + + +
+ {{ 'calculated-fields.metrics.filter' | translate }} +
+ + + + + +
{{ 'api-usage.tbel' | translate }} +
+ + + +
+ +
+
{{ 'calculated-fields.metrics.value-source' | translate }}
+ + + @for (inputType of AggInputTypes; track inputType) { + {{ AggInputTypeTranslations.get(inputType) | translate }} + } + + +
+ @if (this.metricForm.get('input.type').value === AggInputType.key) { +
+
{{ 'calculated-fields.argument-name' | translate }}
+ + +
+ } @else { + +
{{ 'api-usage.tbel' | translate }} +
+ + } + +
+
+
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.scss new file mode 100644 index 0000000000..ae692b6f93 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.scss @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import '../../../../../../../scss/constants'; + +$panel-width: 520px; + +:host { + display: flex; + width: $panel-width; + max-width: 100%; + max-height: 80vh; + + .fixed-title-width { + @media #{$mat-xs} { + min-width: 120px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.ts new file mode 100644 index 0000000000..6fb296a161 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.ts @@ -0,0 +1,176 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit, output } from '@angular/core'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { FormBuilder, FormControl, ValidatorFn, Validators } from '@angular/forms'; +import { charsWithNumRegex } from '@shared/models/regex.constants'; +import { + AggFunction, + AggFunctionTranslations, + AggInputType, + AggInputTypeTranslations, + CalculatedFieldAggMetricValue +} from '@shared/models/calculated-field.models'; +import { delay, map } from 'rxjs/operators'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { EntityFilter } from '@shared/models/query/query.models'; +import { merge, Observable, of } from 'rxjs'; +import { ScriptLanguage } from '@shared/models/rule-node.models'; +import { TbEditorCompleter } from '@shared/models/ace/completion.models'; +import { AceHighlightRules } from '@shared/models/ace/ace.models'; + +interface CalculatedFieldAggMetricValuePanel extends CalculatedFieldAggMetricValue { + allowFilter: boolean; +} + +@Component({ + selector: 'tb-calculated-field-metrics-panel', + templateUrl: './calculated-field-metrics-panel.component.html', + styleUrls: ['./calculated-field-metrics-panel.component.scss'] +}) +export class CalculatedFieldMetricsPanelComponent implements OnInit { + + @Input() buttonTitle: string; + @Input() metric: CalculatedFieldAggMetricValue; + @Input() usedNames: string[]; + @Input() arguments: Array; + @Input() editorCompleter: TbEditorCompleter; + @Input() highlightRules: AceHighlightRules; + + metricDataApplied = output(); + filterExpanded = false; + functionArgs: Array + + metricForm = this.fb.group({ + name: ['', [Validators.required, this.uniqNameRequired(), this.forbiddenNameValidator(), Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], + function: [AggFunction.AVG], + allowFilter: [false], + filter: ['', Validators.required], + input: this.fb.group({ + type: [AggInputType.key], + key: ['', Validators.required], + function: ['', Validators.required], + }) + }); + + entityFilter: EntityFilter; + + readonly AggFunctions = Object.values(AggFunction) as AggFunction[]; + readonly AggFunctionTranslations = AggFunctionTranslations; + readonly ScriptLanguage = ScriptLanguage; + readonly AggInputType = AggInputType; + readonly AggInputTypes = Object.values(AggInputType) as AggInputType[]; + readonly AggInputTypeTranslations = AggInputTypeTranslations; + + constructor( + private fb: FormBuilder, + private popover: TbPopoverComponent + ) { + this.observeUpdatePosition(); + this.observeFilterAllowChange(); + this.observeInputTypeChange(); + } + + ngOnInit(): void { + const data: CalculatedFieldAggMetricValuePanel = { + ...this.metric, + allowFilter: !!this.metric.filter, + } + this.metricForm.patchValue(data, {emitEvent: false}); + + this.validateFilter(data.allowFilter); + this.validateInputTypeFilter(data.input?.type ?? AggInputType.key); + + this.functionArgs = ['ctx', ...this.arguments]; + } + + fetchOptions(searchText: string): Observable> { + const search = searchText ? searchText?.toLowerCase() : ''; + return of(this.arguments).pipe(map(name => name?.filter(option => option.toLowerCase().includes(search)))); + } + + private observeFilterAllowChange(): void { + this.metricForm.get('allowFilter').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(value => this.validateFilter(value)); + } + + private observeInputTypeChange(): void { + this.metricForm.get('input.type').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(value => this.validateInputTypeFilter(value)); + } + + private validateFilter(allowFilter = false): void { + if (allowFilter) { + this.metricForm.get('filter').enable({emitEvent: false}); + } else { + this.metricForm.get('filter').disable({emitEvent: false}); + } + this.filterExpanded = allowFilter; + } + + private validateInputTypeFilter(value: AggInputType): void { + const inputForm = this.metricForm.get('input'); + if (value === AggInputType.key) { + inputForm.get('key').enable({emitEvent: false}); + inputForm.get('function').disable({emitEvent: false}); + } else { + inputForm.get('key').disable({emitEvent: false}); + inputForm.get('function').enable({emitEvent: false}); + } + } + + saveZone(): void { + const value = this.metricForm.value as CalculatedFieldAggMetricValuePanel; + if (!value.allowFilter) { + delete value.filter; + } + delete value.allowFilter; + this.metricDataApplied.emit(value); + } + + cancel(): void { + this.popover.hide(); + } + + private uniqNameRequired(): ValidatorFn { + return (control: FormControl) => { + const newName = control.value.trim().toLowerCase(); + const isDuplicate = this.usedNames?.some(name => name.toLowerCase() === newName); + + return isDuplicate ? { duplicateName: true } : null; + }; + } + + private forbiddenNameValidator(): ValidatorFn { + return (control: FormControl) => { + const trimmedValue = control.value.trim().toLowerCase(); + const forbiddenNames = ['ctx', 'e', 'pi']; + return forbiddenNames.includes(trimmedValue) ? { forbiddenName: true } : null; + }; + } + + private observeUpdatePosition(): void { + merge( + this.metricForm.get('allowFilter').valueChanges, + this.metricForm.get('input.type').valueChanges + ) + .pipe(delay(50), takeUntilDestroyed()) + .subscribe(() => this.popover.updatePosition()); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.html new file mode 100644 index 0000000000..481ad9ee13 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.html @@ -0,0 +1,111 @@ + +
+
+ + + +
{{ 'calculated-fields.metrics.metric-name' | translate }}
+ + +
+
{{ metric.name }}
+ +
+ + + + + {{ 'calculated-fields.metrics.aggregation' | translate }} + + +
{{ AggFunctionTranslations.get(metric.function) | translate }}
+ + + + + {{ 'calculated-fields.metrics.filtered' | translate }} + + + {{ metric.filter ? 'check_box' : 'check_box_outline_blank' }} + + + + + {{ 'calculated-fields.metrics.value-source' | translate }} + + +
{{ AggInputTypeTranslations.get(metric.input.type) | translate }}
+ + + + + + +
+ + +
+ + + + +
+
+ {{ 'calculated-fields.metrics.no-metrics-configured' | translate }} +
+ @if (errorText) { + + } +
+
+ + @if (maxArgumentsPerCF && metricsFormArray.length >= maxArgumentsPerCF) { +
+ warning + {{ 'calculated-fields.metrics.max-metrics' | translate }} +
+ } +
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.scss new file mode 100644 index 0000000000..430958d0f4 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.scss @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .arguments-table { + min-height: 108px; + + &-with-error { + min-height: 150px; + } + + .mat-mdc-table { + table-layout: fixed; + } + + .key-text { + font-size: 13px; + } + + .copy-argument-name { + visibility: hidden; + transition: visibility 0.1s; + } + + .argument-name-cell:hover { + .copy-argument-name { + visibility: visible; + } + } + } + + .max-args-warning { + .mat-icon { + color: #FAA405; + } + } + + .tb-form-table-row-cell-buttons { + --mat-badge-legacy-small-size-container-size: 8px; + --mat-badge-small-size-container-overlap-offset: -5px; + --mat-badge-small-size-text-size: 0; + } +} + +:host ::ng-deep { + .arguments-table:not(.arguments-table-with-error) { + .mdc-data-table__row:last-child .mat-mdc-cell { + border-bottom: none; + } + } + + .arguments-table { + .mat-mdc-header-row.mat-row-select .mat-mdc-header-cell.entity-type-header { + padding: 0 28px 0 0; + } + } + + .copy-argument-name { + .mat-icon { + font-size: 16px; + padding: 4px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.ts new file mode 100644 index 0000000000..2e4ac30165 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.ts @@ -0,0 +1,242 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AfterViewInit, + ChangeDetectorRef, + Component, + DestroyRef, + forwardRef, + Input, + Renderer2, + ViewChild, + ViewContainerRef, +} from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, +} from '@angular/forms'; +import { + AggFunctionTranslations, + AggInputTypeTranslations, + CalculatedFieldAggMetric, + CalculatedFieldAggMetricValue, +} from '@shared/models/calculated-field.models'; +import { MatButton } from '@angular/material/button'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { isDefinedAndNotNull, isEqual } from '@core/utils'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { TbTableDatasource } from '@shared/components/table/table-datasource.abstract'; +import { MatSort, SortDirection } from '@angular/material/sort'; +import { getCurrentAuthState } from '@core/auth/auth.selectors'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { + CalculatedFieldMetricsPanelComponent +} from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component'; +import { TbEditorCompleter } from '@shared/models/ace/completion.models'; +import { AceHighlightRules } from '@shared/models/ace/ace.models'; + +@Component({ + selector: 'tb-calculated-field-metrics-table', + templateUrl: './calculated-field-metrics-table.component.html', + styleUrls: [`calculated-field-metrics-table.component.scss`], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => CalculatedFieldMetricsTableComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => CalculatedFieldMetricsTableComponent), + multi: true + } + ], +}) +export class CalculatedFieldMetricsTableComponent implements ControlValueAccessor, Validator, AfterViewInit { + + @Input() arguments: Array; + @Input() editorCompleter: TbEditorCompleter; + @Input() highlightRules: AceHighlightRules; + + @ViewChild(MatSort, { static: true }) sort: MatSort; + + errorText = ''; + metricsFormArray = this.fb.array([]); + sortOrder = { direction: 'asc' as SortDirection, property: '' }; + dataSource = new CalculatedFieldMetricsDatasource(); + + displayColumns = ['name', 'function', 'filter', 'valueSource', 'actions'] + + readonly AggFunctionTranslations = AggFunctionTranslations; + readonly AggInputTypeTranslations = AggInputTypeTranslations; + readonly maxArgumentsPerCF = getCurrentAuthState(this.store).maxArgumentsPerCF - 2; + + private popoverComponent: TbPopoverComponent; + private propagateChange: (zonesObj: Record) => void = () => {}; + + constructor( + private fb: FormBuilder, + private popoverService: TbPopoverService, + private viewContainerRef: ViewContainerRef, + private cd: ChangeDetectorRef, + private renderer: Renderer2, + private destroyRef: DestroyRef, + private store: Store + ) { + this.metricsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(value => { + this.updateDataSource(value); + this.propagateChange(this.getMetricsObject(value)); + }); + } + + ngAfterViewInit(): void { + this.sort.sortChange.asObservable().pipe( + takeUntilDestroyed(this.destroyRef) + ).subscribe(() => { + this.sortOrder.property = this.sort.active; + this.sortOrder.direction = this.sort.direction; + this.updateDataSource(this.metricsFormArray.value); + }); + } + + registerOnChange(fn: (zonesObj: Record) => void): void { + this.propagateChange = fn; + } + + registerOnTouched(_fn: any): void {} + + validate(): ValidationErrors | null { + this.updateErrorText(); + return this.errorText ? { metricsFormArray: false } : null; + } + + onDelete($event: Event, metric: CalculatedFieldAggMetricValue): void { + $event.stopPropagation(); + const index = this.metricsFormArray.controls.findIndex(control => isEqual(control.value, metric)); + this.metricsFormArray.removeAt(index); + this.metricsFormArray.markAsDirty(); + } + + manageMetrics($event: Event, matButton: MatButton, metric = {} as CalculatedFieldAggMetricValue): void { + $event?.stopPropagation(); + if (this.popoverComponent && !this.popoverComponent.tbHidden) { + this.popoverComponent.hide(); + } + const trigger = matButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const index = this.metricsFormArray.controls.findIndex(control => isEqual(control.value, metric)); + const isExists = index !== -1; + const ctx = { + index, + metric, + buttonTitle: isExists ? 'action.apply' : 'action.add', + usedNames: this.metricsFormArray.value.map(({ name }) => name).filter(name => name !== metric.name), + arguments: this.arguments, + editorCompleter: this.editorCompleter, + highlightRules: this.highlightRules + }; + this.popoverComponent = this.popoverService.displayPopover({ + trigger, + renderer: this.renderer, + componentType: CalculatedFieldMetricsPanelComponent, + hostView: this.viewContainerRef, + preferredPlacement: isExists ? ['leftOnly', 'leftTopOnly', 'leftBottomOnly'] : ['rightOnly', 'rightTopOnly', 'rightBottomOnly'], + context: ctx, + isModal: true + }); + this.popoverComponent.tbComponentRef.instance.metricDataApplied.subscribe((value) => { + this.popoverComponent.hide(); + if (isExists) { + this.metricsFormArray.at(index).setValue(value); + } else { + this.metricsFormArray.push(this.fb.control(value)); + } + this.cd.markForCheck(); + }); + } + } + + private updateDataSource(value: CalculatedFieldAggMetricValue[]): void { + const sortedValue = this.sortData(value); + this.dataSource.loadData(sortedValue); + } + + private updateErrorText(): void { + if (!this.metricsFormArray.controls.length) { + this.errorText = 'calculated-fields.metrics.metrics-empty'; + } else { + this.errorText = ''; + } + } + + private getMetricsObject(value: CalculatedFieldAggMetricValue[]): Record { + return value.reduce((acc, metricValue) => { + const { name, ...metric } = metricValue; + acc[name] = metric; + return acc; + }, {} as Record); + } + + writeValue(metrics: Record): void { + this.metricsFormArray.clear(); + this.populateZonesFormArray(metrics); + } + + private populateZonesFormArray(metrics: Record): void { + Object.keys(metrics).forEach(key => { + const value: CalculatedFieldAggMetricValue = { + ...metrics[key], + name: key + }; + this.metricsFormArray.push(this.fb.control(value), { emitEvent: false }); + }); + this.metricsFormArray.updateValueAndValidity(); + } + + private getSortValue(metric: CalculatedFieldAggMetricValue, column: string): string { + switch (column) { + case 'function': + return metric.function; + case 'filter': + return isDefinedAndNotNull(metric.filter).toString(); + default: + return metric.name; + } + } + + private sortData(data: CalculatedFieldAggMetricValue[]): CalculatedFieldAggMetricValue[] { + return data.sort((a, b) => { + const valA = this.getSortValue(a, this.sortOrder.property) ?? ''; + const valB = this.getSortValue(b, this.sortOrder.property) ?? ''; + return (this.sortOrder.direction === 'asc' ? 1 : -1) * valA.localeCompare(valB); + }); + } +} + +class CalculatedFieldMetricsDatasource extends TbTableDatasource { + constructor() { + super(); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html index 1988141c2a..ce2d22fbbc 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html @@ -51,8 +51,13 @@
- {{ 'calculated-fields.metrics' | translate }} + {{ 'calculated-fields.metrics.metrics' | translate }}
+ ({ scope: AttributeScope.SERVER_SCOPE, @@ -93,8 +94,8 @@ export class RelatedEntitiesAggregationComponentComponent implements ControlValu readonly minAllowedDeduplicationIntervalInSecForCF = getCurrentAuthState(this.store).minAllowedDeduplicationIntervalInSecForCF; - functionArgs$ = this.relatedAggregationConfiguration.get('arguments').valueChanges.pipe( - map(argumentsObj => ['ctx', ...Object.keys(argumentsObj)]) + arguments$ = this.relatedAggregationConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => Object.keys(argumentsObj)) ); argumentsEditorCompleter$ = this.relatedAggregationConfiguration.get('arguments').valueChanges.pipe( diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts index a2c48c1896..b272f1e965 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts @@ -26,6 +26,12 @@ import { import { RelatedEntitiesAggregationComponentComponent } from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component'; +import { + CalculatedFieldMetricsTableComponent +} from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component'; +import { + CalculatedFieldMetricsPanelComponent +} from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component'; @NgModule({ imports: [ @@ -36,6 +42,8 @@ import { ], declarations: [ RelatedEntitiesAggregationComponentComponent, + CalculatedFieldMetricsTableComponent, + CalculatedFieldMetricsPanelComponent ], exports: [ RelatedEntitiesAggregationComponentComponent, diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 8ea6659bca..377b443fb6 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -113,6 +113,7 @@ export interface CalculatedFieldRelatedAggregationConfiguration { type: CalculatedFieldType.RELATED_ENTITIES_AGGREGATION; relation: RelationPathLevel; arguments: Record; + metrics: Record; deduplicationIntervalInSec: number; useLatestTs: boolean; output: Omit; @@ -251,6 +252,54 @@ export interface CalculatedFieldArgument { timeWindow?: number; } +export enum AggFunction { + AVG='AVG', + MIN='MIN', + MAX='MAX', + SUM='SUM', + COUNT='COUNT', + COUNT_UNIQUE='COUNT_UNIQUE' +} + +export const AggFunctionTranslations = new Map([ + [AggFunction.AVG, 'calculated-fields.metrics.aggregation-type.avg'], + [AggFunction.MIN, 'calculated-fields.metrics.aggregation-type.min'], + [AggFunction.MAX, 'calculated-fields.metrics.aggregation-type.max'], + [AggFunction.SUM, 'calculated-fields.metrics.aggregation-type.sum'], + [AggFunction.COUNT, 'calculated-fields.metrics.aggregation-type.count'], + [AggFunction.COUNT_UNIQUE, 'calculated-fields.metrics.aggregation-type.count-unique'], +]) + +export interface CalculatedFieldAggMetric { + function: AggFunction; + filter?: string; + input: AggKeyInput | AggFunctionInput; +} + +export interface CalculatedFieldAggMetricValue extends CalculatedFieldAggMetric { + name: string; +} + +export enum AggInputType { + key = 'key', + function = 'function' +} + +export const AggInputTypeTranslations = new Map([ + [AggInputType.key, 'calculated-fields.metrics.value-source-type.key'], + [AggInputType.function, 'calculated-fields.metrics.value-source-type.function'], +]) + +export interface AggKeyInput { + type: AggInputType.key; + key: string; +} + +export interface AggFunctionInput { + type: AggInputType.function; + function: string; +} + export interface CalculatedFieldGeofencing { perimeterKeyName: string; reportStrategy: GeofencingReportStrategy; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 0333c5ed2e..15207999f6 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1159,10 +1159,36 @@ "output-key": "Output key", "copy-output-key": "Copy output key", "aggregation-path-related-entities": "Aggregation path to related entities", - "metrics": "Metrics", "deduplication-interval": "Deduplication interval", "deduplication-interval-min": "Deduplication interval should be at least {{ sec }} second.", "deduplication-interval-required": "Deduplication interval is required.", + "metrics": { + "metrics": "Metrics", + "metrics-empty": "At least one metric must be configured.", + "metric-name": "Metric name", + "copy-metric-name": "Copy metric name", + "aggregation": "Aggregation", + "aggregation-type": { + "avg": "Average", + "min": "Minimum", + "max": "Maximum", + "sum": "Sum", + "count": "Count", + "count-unique": "Count unique" + }, + "filtered": "Filtered", + "value-source": "Value source", + "value-source-type": { + "key": "Key", + "function": "Function" + }, + "no-metrics-configured": "No metrics configured", + "add-metric": "Add metric", + "max-metrics": "Maximum number of metrics reached.", + "metric-settings": "Metric settings", + "filter": "Filter", + "filter-hint": "Enables filtering of entities during aggregation. The filter function must return a boolean value and can use all configured arguments." + }, "hint": { "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with time series rolling type.", "arguments-propagate-arguments-with-rolling": "'Time series rolling' type is incompatible with 'Arguments only' propagation.", @@ -1182,7 +1208,7 @@ "output-key-max-length": "Output key should be less than 256 characters.", "output-key-forbidden": "Output key is reserved and cannot be used.", "entity-type-required": "Entity type is required", - "name-required": "Mame is required.", + "name-required": "Name is required.", "name-pattern": "Name is invalid.", "name-duplicate": "Name with such name already exists.", "name-max-length": "Name should be less than 256 characters.", From 522369383b94d2b6b46cd3c377c6e28bca83022b Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 24 Oct 2025 19:31:06 +0300 Subject: [PATCH 25/33] UI: Improvement popover in cf; Simplify style --- .../calculated-fields-table-config.ts | 4 +- ...ulated-field-argument-panel.component.html | 8 +- ...ulated-field-argument-panel.component.scss | 19 +- ...lculated-field-argument-panel.component.ts | 19 +- .../common/calculated-field-panel.scss | 58 +++ ...eofencing-zone-groups-panel.component.html | 421 +++++++++--------- ...eofencing-zone-groups-panel.component.scss | 20 +- ...-geofencing-zone-groups-panel.component.ts | 33 +- ...eofencing-zone-groups-table.component.html | 2 +- ...eofencing-zone-groups-table.component.scss | 76 ---- ...-geofencing-zone-groups-table.component.ts | 6 +- .../calculated-field-output.component.html | 2 +- ...culated-field-metrics-panel.component.html | 220 ++++----- ...alculated-field-metrics-panel.component.ts | 16 +- ...culated-field-metrics-table.component.html | 20 +- ...culated-field-metrics-table.component.scss | 76 ---- ...alculated-field-metrics-table.component.ts | 4 +- ...ities-aggregation-component.component.html | 4 +- ...ties-aggregation-component.component.scss} | 21 +- ...ntities-aggregation-component.component.ts | 1 + 20 files changed, 431 insertions(+), 599 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/common/calculated-field-panel.scss delete mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/calculated-field-geofencing-zone-groups-table.component.scss delete mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.scss rename ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/{calculated-field-metrics-panel.component.scss => related-entities-aggregation-component.component.scss} (73%) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 4104b97221..1b48771247 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -111,7 +111,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig('expression', 'calculated-fields.expression', '300px'); + const expressionColumn = new EntityTableColumn('expression', 'calculated-fields.expression', '250px'); expressionColumn.sortable = false; expressionColumn.cellContentFunction = entity => { const expressionLabel = this.getExpressionLabel(entity); @@ -124,7 +124,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig('createdTime', 'common.created-time', this.datePipe, '150px')); this.columns.push(new EntityTableColumn('name', 'common.name', '33%')); - this.columns.push(new EntityTableColumn('type', 'common.type', '80px', entity => this.translate.instant(CalculatedFieldTypeTranslations.get(entity.type)))); + this.columns.push(new EntityTableColumn('type', 'common.type', '170px', entity => this.translate.instant(CalculatedFieldTypeTranslations.get(entity.type)), () => ({whiteSpace: 'nowrap' }))); this.columns.push(expressionColumn); this.cellActionDescriptors.push( diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html index 607b107094..1ffa7ccf77 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html @@ -15,9 +15,9 @@ limitations under the License. --> -
-
-
{{ 'calculated-fields.argument-settings' | translate }}
+
+
{{ 'calculated-fields.metrics.metric-settings' | translate }}
+
@if (hint) {
{{ hint | translate }} @@ -190,7 +190,7 @@ }
-
+
-
-
-
{{ $index+1 }}
- - - @for (direction of GeofencingDirectionList; track direction) { - {{ GeofencingDirectionLevelTranslations.get(direction) | translate }} - } - - - - -
-
- -
-
- } -
- } @else { - {{ 'calculated-fields.no-level' | translate }} - } - @if (levelsFormArray().errors) { - - } -
-
- @if (maxRelationLevelPerCfArgument && levelsFormArray().length >= maxRelationLevelPerCfArgument) { -
- warning - {{ 'calculated-fields.max-allowed-levels-error' | translate }} -
- } @else { - - } -
- +
{{ ArgumentEntityTypeParamsMap.get(entityType).title | translate }}
+
- - - @if (entityFilter.singleEntity?.id) { -
-
- {{ 'calculated-fields.perimeter-attribute-key' | translate }} + } + + +
+ + {{ 'calculated-fields.entity-zone-relationship' | translate }} +
+
+
+
calculated-fields.level
+
calculated-fields.direction-level
+
calculated-fields.relation-type
+
- @if (entityType === ArgumentEntityType.RelationQuery) { - - - @if (geofencingFormGroup.get('perimeterKeyName').touched && geofencingFormGroup.get('perimeterKeyName').hasError('required')) { - - warning - - } @else if (geofencingFormGroup.get('perimeterKeyName').touched && geofencingFormGroup.get('perimeterKeyName').hasError('pattern')) { - - warning - + @if (levelsFormArray()?.controls?.length) { +
+ @for (keyControl of levelsFormArray().controls; track trackByKey; ) { +
+
+ +
+
+
{{ $index + 1 }}
+ + + @for (direction of GeofencingDirectionList; track direction) { + {{ GeofencingDirectionLevelTranslations.get(direction) | translate }} + } + + + + +
+
+ +
+
} - +
} @else { - + {{ 'calculated-fields.no-level' | translate }} + } + @if (levelsFormArray().errors) { + }
- } +
+ @if (maxRelationLevelPerCfArgument && levelsFormArray().length >= maxRelationLevelPerCfArgument) { +
+ warning + {{ 'calculated-fields.max-allowed-levels-error' | translate }} +
+ } @else { + + } +
+ +
+ + + @if (entityFilter.singleEntity?.id) {
-
{{ 'calculated-fields.report-strategy' | translate }}
- - - @for (strategy of GeofencingReportStrategyList; track strategy) { - {{ GeofencingReportStrategyTranslations.get(strategy) | translate }} - } - - -
- -
- -
- {{ 'calculated-fields.create-relation-with-matched-zones' | translate }} +
+ {{ 'calculated-fields.perimeter-attribute-key' | translate }}
- -
-
{{ 'calculated-fields.direction' | translate }}
- - - @for (direction of GeofencingDirectionList; track direction) { - {{ GeofencingDirectionTranslations.get(direction) | translate }} + @if (entityType === ArgumentEntityType.RelationQuery) { + + + @if (geofencingFormGroup.get('perimeterKeyName').touched && geofencingFormGroup.get('perimeterKeyName').hasError('required')) { + + warning + + } @else if (geofencingFormGroup.get('perimeterKeyName').touched && geofencingFormGroup.get('perimeterKeyName').hasError('pattern')) { + + warning + } - - + + } @else { + + }
-
-
{{ 'calculated-fields.relation-type' | translate }}
- - + } +
+
{{ 'calculated-fields.report-strategy' | translate }}
+ + + @for (strategy of GeofencingReportStrategyList; track strategy) { + {{ GeofencingReportStrategyTranslations.get(strategy) | translate }} + } + + +
+ +
+ +
+ {{ 'calculated-fields.create-relation-with-matched-zones' | translate }}
+ +
+
{{ 'calculated-fields.direction' | translate }}
+ + + @for (direction of GeofencingDirectionList; track direction) { + {{ GeofencingDirectionTranslations.get(direction) | translate }} + } + + +
+
+
{{ 'calculated-fields.relation-type' | translate }}
+ +
-
+
@if (simpleMode) { @if (hiddenName) { -
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html index 5c2985321a..b7c58f7633 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html @@ -15,23 +15,23 @@ limitations under the License. --> -
-
-
{{ 'calculated-fields.metrics.metric-settings' | translate }}
-
-
-
{{ 'calculated-fields.metrics.metric-name' | translate }}
- - - @if (metricForm.get('name').touched && metricForm.get('name').hasError('required')) { - - warning - - } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('duplicateName')) { +
+
{{ 'calculated-fields.metrics.metric-settings' | translate }}
+
+
+
{{ 'calculated-fields.metrics.metric-name' | translate }}
+ + + @if (metricForm.get('name').touched && metricForm.get('name').hasError('required')) { + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('duplicateName')) { } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('pattern')) { - - warning - - } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('maxlength')) { - - warning - - } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('forbiddenName')) { - - warning - + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('maxlength')) { + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('forbiddenName')) { + + warning + + } + +
+
+
{{ 'calculated-fields.metrics.aggregation' | translate }}
+ + + @for (aggFunction of AggFunctions; track aggFunction) { + {{ AggFunctionTranslations.get(aggFunction) | translate }} } - -
-
-
{{ 'calculated-fields.metrics.aggregation' | translate }}
- - - @for (aggFunction of AggFunctions; track aggFunction) { - {{ AggFunctionTranslations.get(aggFunction) | translate }} - } - - -
+ + +
-
- - - - -
- {{ 'calculated-fields.metrics.filter' | translate }} -
- - - - - -
{{ 'api-usage.tbel' | translate }} +
+ + + + +
+ {{ 'calculated-fields.metrics.filter' | translate }}
- - - -
- -
-
{{ 'calculated-fields.metrics.value-source' | translate }}
- - - @for (inputType of AggInputTypes; track inputType) { - {{ AggInputTypeTranslations.get(inputType) | translate }} - } - - -
- @if (this.metricForm.get('input.type').value === AggInputType.key) { -
-
{{ 'calculated-fields.argument-name' | translate }}
- - -
- } @else { + + + + {{ 'api-usage.tbel' | translate }}
- } - + +
+ +
+
{{ 'calculated-fields.metrics.value-source' | translate }}
+ + + @for (inputType of AggInputTypes; track inputType) { + {{ AggInputTypeTranslations.get(inputType) | translate }} + } + + +
+ @if (this.metricForm.get('input.type').value === AggInputType.key) { +
+
{{ 'calculated-fields.argument-name' | translate }}
+ + +
+ } @else { + +
{{ 'api-usage.tbel' | translate }} +
+ + } +
-
+