From f667f0f23f796492050e336f1fee37e8495a69d0 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 10 Oct 2025 16:58:00 +0300 Subject: [PATCH] (wip) aggregation cf --- .../CalculatedFieldEntityActor.java | 3 + ...CalculatedFieldEntityMessageProcessor.java | 154 +++++++-- ...alculatedFieldManagerMessageProcessor.java | 320 +++++++++++++++++- .../CalculatedFieldRelatedEntityMsg.java | 49 +++ ...tractCalculatedFieldProcessingService.java | 97 ++++++ .../service/cf/CalculatedFieldCache.java | 3 + .../cf/CalculatedFieldProcessingService.java | 2 + .../cf/DefaultCalculatedFieldCache.java | 28 ++ ...faultCalculatedFieldProcessingService.java | 8 + .../DefaultCalculatedFieldQueueService.java | 39 ++- .../service/cf/ctx/state/ArgumentEntry.java | 10 +- .../cf/ctx/state/ArgumentEntryType.java | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 2 +- .../cf/ctx/state/CalculatedFieldCtx.java | 140 ++++++++ .../cf/ctx/state/CalculatedFieldState.java | 5 +- .../ctx/state/SingleValueArgumentEntry.java | 8 +- .../state/aggregation/AggArgumentEntry.java | 72 ++++ .../aggregation/AggSingleArgumentEntry.java | 83 +++++ ...ValuesAggregationCalculatedFieldState.java | 161 +++++++++ .../service/cf/ctx/state/aggregation/agg.json | 65 ++++ .../state/aggregation/function/AggEntry.java | 45 +++ .../function/AggFunctionFactory.java | 33 ++ .../aggregation/function/AvgAggEntry.java | 45 +++ .../aggregation/function/BaseAggEntry.java | 52 +++ .../aggregation/function/CountAggEntry.java | 41 +++ .../function/CountUniqueAggEntry.java | 45 +++ .../aggregation/function/MaxAggEntry.java | 40 +++ .../aggregation/function/MinAggEntry.java | 40 +++ .../aggregation/function/SumAggEntry.java | 42 +++ .../entitiy/EntityStateSourcingListener.java | 11 + .../queue/DefaultTbClusterService.java | 40 ++- .../processing/AbstractConsumerService.java | 18 +- .../utils/CalculatedFieldArgumentUtils.java | 2 + .../server/utils/CalculatedFieldUtils.java | 48 ++- ...tValuesAggregationCalculatedFieldTest.java | 314 +++++++++++++++++ .../server/controller/AbstractWebTest.java | 10 + .../server/cluster/TbClusterService.java | 8 + .../server/dao/relation/RelationService.java | 8 + .../common/data/cf/CalculatedFieldType.java | 3 +- .../CalculatedFieldConfiguration.java | 4 +- .../aggregation/AggFunction.java | 20 ++ .../aggregation/AggFunctionInput.java | 34 ++ .../configuration/aggregation/AggInput.java | 36 ++ .../aggregation/AggKeyInput.java | 34 ++ .../configuration/aggregation/AggMetric.java | 31 ++ .../configuration/aggregation/AggSource.java | 30 ++ .../aggregation/CfAggTrigger.java | 104 ++++++ ...gregationCalculatedFieldConfiguration.java | 52 +++ .../server/common/msg/MsgType.java | 2 + .../msg/plugin/ComponentLifecycleMsg.java | 6 +- .../server/common/util/ProtoUtils.java | 2 + common/proto/src/main/proto/queue.proto | 7 + .../script/api/tbel/TbelCfArg.java | 1 + .../tbel/TbelCfLatestValuesAggregation.java | 44 +++ .../dao/relation/BaseRelationService.java | 34 ++ .../server/dao/relation/RelationCacheKey.java | 7 +- .../server/dao/relation/RelationDao.java | 4 + .../dao/sql/relation/JpaRelationDao.java | 10 + .../dao/sql/relation/RelationRepository.java | 39 ++- 59 files changed, 2527 insertions(+), 70 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java create mode 100644 application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunction.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunctionInput.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggKeyInput.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java index cababd4b6d..ed4131c114 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -73,6 +73,9 @@ public class CalculatedFieldEntityActor extends AbstractCalculatedFieldActor { case CF_ENTITY_DELETE_MSG: processor.process((CalculatedFieldEntityDeleteMsg) msg); break; + case CF_RELATED_ENTITY_MSG: + processor.process((CalculatedFieldRelatedEntityMsg) msg); + break; case CF_ENTITY_TELEMETRY_MSG: processor.process((EntityCalculatedFieldTelemetryMsg) msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index ccbdcc6b33..98b5eba4c8 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -52,6 +52,9 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -66,7 +69,9 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createStateByType; @@ -189,7 +194,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } - public void process(CalculatedFieldEntityDeleteMsg msg) { + public void process(CalculatedFieldEntityDeleteMsg msg) throws CalculatedFieldException { log.debug("[{}] Processing CF entity delete msg.", msg.getEntityId()); if (this.entityId.equals(msg.getEntityId())) { if (states.isEmpty()) { @@ -200,16 +205,74 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM actorCtx.stop(actorCtx.getSelf()); } } else { - var cfId = new CalculatedFieldId(msg.getEntityId().getId()); - var state = removeState(cfId); - if (state != null) { - cfStateService.deleteState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); + EntityId msgEntityId = msg.getEntityId(); + if (msgEntityId instanceof CalculatedFieldId cfId) { + var state = removeState(cfId); + if (state != null) { + cfStateService.deleteState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); + } else { + msg.getCallback().onSuccess(); + } } else { - msg.getCallback().onSuccess(); + if (states.isEmpty()) { + msg.getCallback().onSuccess(); + } + for (Map.Entry entry : states.entrySet()) { + LatestValuesAggregationCalculatedFieldState state = (LatestValuesAggregationCalculatedFieldState) entry.getValue(); + state.getArguments().forEach((argName, argEntry) -> { + AggArgumentEntry aggArgEntry = (AggArgumentEntry) argEntry; + aggArgEntry.getAggInputs().remove(msgEntityId); + }); + state.getInputs().remove(msgEntityId); + state.setLastMetricsEvalTs(-1); + processStateIfReady(state, Collections.emptyMap(), state.getCtx(), Collections.emptyList(), null, null, msg.getCallback()); + } } } } + public void process(CalculatedFieldRelatedEntityMsg msg) throws CalculatedFieldException { + log.debug("[{}] Processing CF related entity msg.", msg.getEntityId()); + CalculatedFieldCtx cfCtx = msg.getCalculatedField(); + var state = states.get(cfCtx.getCfId()); + Map fetchedArguments = fetchAggArguments(msg.getCalculatedField(), msg.getEntityId()); + try { + if (state == null) { + state = createState(cfCtx); + } else { + state.setCtx(cfCtx, actorCtx); + } + if (state.isSizeOk()) { + if (state instanceof LatestValuesAggregationCalculatedFieldState latestValuesState) { + latestValuesState.setLastMetricsEvalTs(-1); + } + state.update(fetchedArguments, cfCtx); + state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, cfCtx.getCfId(), entityId), cfCtx.getMaxStateSize()); + states.put(cfCtx.getCfId(), state); + processStateIfReady(state, fetchedArguments, cfCtx, Collections.singletonList(cfCtx.getCfId()), null, null, msg.getCallback()); + } else { + throw new RuntimeException(cfCtx.getSizeExceedsLimitMessage()); + } + } catch (Exception e) { + log.debug("[{}][{}] Failed to initialize CF state", entityId, cfCtx.getCfId(), e); + if (e instanceof CalculatedFieldException cfe) { + throw cfe; + } + throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(entityId).cause(e).build(); + } + } + + @SneakyThrows + private Map fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { + ListenableFuture> argumentsFuture = cfService.fetchAggArguments(ctx, entityId); + // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. + // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. + // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, + // but this will significantly complicate the code. + return argumentsFuture.get(1, TimeUnit.MINUTES); + } + + public void process(EntityCalculatedFieldTelemetryMsg msg) throws CalculatedFieldException { log.trace("[{}] Processing CF telemetry msg: {}", msg.getEntityId(), msg); var proto = msg.getProto(); @@ -462,59 +525,78 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { - return mapToArguments(ctx.getMainEntityArguments(), data); + return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getAggregationInputs(), data); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { - return mapToArguments(ctx.getLinkedAndDynamicArgs(entityId), data); + return mapToArguments(entityId, ctx.getLinkedAndDynamicArgs(entityId), ctx.getAggregationInputs(), data); } - private Map mapToArguments(Map argNames, List data) { - if (argNames.isEmpty()) { - return Collections.emptyMap(); - } + private Map mapToArguments(EntityId originator, Map argNames, Map aggArgNames, List data) { Map arguments = new HashMap<>(); - for (TsKvProto item : data) { - ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); - String argName = argNames.get(key); - if (argName != null) { - arguments.put(argName, new SingleValueArgumentEntry(item)); + if (!aggArgNames.isEmpty()) { + for (Map.Entry entry : aggArgNames.entrySet()) { + for (TsKvProto item : data) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); + if (key.equals(entry.getValue())) { + arguments.put(entry.getKey(), new AggSingleArgumentEntry(originator, item)); + } + } } - key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); - argName = argNames.get(key); - if (argName != null) { - arguments.put(argName, new SingleValueArgumentEntry(item)); + } + if (!argNames.isEmpty()) { + for (TsKvProto item : data) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); + String argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); + argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } } } return arguments; } private Map mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List attrDataList) { - return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), scope, attrDataList); + return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), ctx.getAggregationInputs(), scope, attrDataList); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { var argNames = ctx.getLinkedAndDynamicArgs(entityId); - if (argNames.isEmpty()) { - return Collections.emptyMap(); - } List geofencingArgumentNames = ctx.getLinkedEntityAndCurrentOwnerGeofencingArgumentNames(); - return mapToArguments(entityId, argNames, geofencingArgumentNames, scope, attrDataList); + Map aggregationInputs = ctx.getAggregationInputs(); + return mapToArguments(entityId, argNames, geofencingArgumentNames, aggregationInputs, scope, attrDataList); } - private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, AttributeScopeProto scope, List attrDataList) { + private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, Map aggArgNames, AttributeScopeProto scope, List attrDataList) { Map arguments = new HashMap<>(); - for (AttributeValueProto item : attrDataList) { - ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); - String argName = argNames.get(key); - if (argName == null) { - continue; + if (!argNames.isEmpty()) { + for (AttributeValueProto item : attrDataList) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + String argName = argNames.get(key); + if (argName == null) { + continue; + } + if (geofencingArgNames.contains(argName)) { + arguments.put(argName, new GeofencingArgumentEntry(entityId, item)); + continue; + } + arguments.put(argName, new SingleValueArgumentEntry(item)); } - if (geofencingArgNames.contains(argName)) { - arguments.put(argName, new GeofencingArgumentEntry(entityId, item)); - continue; + } + if (!aggArgNames.isEmpty()) { + for (AttributeValueProto item : attrDataList) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + for (Map.Entry entry : aggArgNames.entrySet()) { + if (key.equals(entry.getValue())) { + arguments.put(entry.getKey(), new AggSingleArgumentEntry(entityId, item)); + } + } } - arguments.put(argName, new SingleValueArgumentEntry(item)); } return arguments; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 4675821a5b..8a717c3cdb 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -25,18 +25,27 @@ import org.thingsboard.server.actors.calculatedField.EntityInitCalculatedFieldMs import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; +import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.msg.CalculatedFieldStatePartitionRestoreMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; @@ -44,10 +53,13 @@ import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldStateService; @@ -64,6 +76,8 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ScheduledFuture; @@ -82,6 +96,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final Map> entityIdCalculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFieldLinks = new HashMap<>(); private final Map> ownerEntities = new HashMap<>(); + private final Map cfTriggers = new HashMap<>(); private ScheduledFuture cfsReevaluationTask; private final CalculatedFieldProcessingService cfExecService; @@ -90,6 +105,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final DeviceService deviceService; private final AssetService assetService; private final CustomerService customerService; + private final RelationService relationService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; private final TenantEntityProfileCache entityProfileCache; @@ -107,6 +123,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware this.deviceService = systemContext.getDeviceService(); this.assetService = systemContext.getAssetService(); this.customerService = systemContext.getCustomerService(); + this.relationService = systemContext.getRelationService(); this.assetProfileCache = systemContext.getAssetProfileCache(); this.deviceProfileCache = systemContext.getDeviceProfileCache(); this.entityProfileCache = new TenantEntityProfileCache(); @@ -129,6 +146,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsReevaluationTask.cancel(true); cfsReevaluationTask = null; } + cfTriggers.clear(); ctx.stop(ctx.getSelf()); } @@ -233,6 +251,20 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware entityProfileCache.add(profileId, entityId); } updateEntityOwner(entityId); + + MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); + + // process aggregation cfs(in any) + List cfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(entityId, profileId); + if (!cfsRelatedToEntity.isEmpty()) { + MultipleTbCallback multiCallback = new MultipleTbCallback(cfsRelatedToEntity.size(), callbackFor2); + cfsRelatedToEntity.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> initRelatedEntity(id, entityId, ctx, cb)); + }); + } else { + callbackFor2.onSuccess(); + } + if (!isMyPartition(entityId, callback)) { return; } @@ -240,11 +272,11 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var profileIdFields = getCalculatedFieldsByEntityId(profileId); var fieldsCount = entityIdFields.size() + profileIdFields.size(); if (fieldsCount > 0) { - MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callbackFor2); entityIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); profileIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); } else { - callback.onSuccess(); + callbackFor2.onSuccess(); } } @@ -254,33 +286,180 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (!isMyPartition(msg.getEntityId(), callback)) { return; } + MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); + + // process aggregation cfs(in any) + List oldCfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getOldProfileId()); + List newCfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getProfileId()); + var fieldsWithRelatedEntityCount = oldCfsRelatedToEntity.size() + newCfsRelatedToEntity.size(); + if (fieldsWithRelatedEntityCount > 0) { + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsWithRelatedEntityCount, callbackFor2); + var entityId = msg.getEntityId(); + oldCfsRelatedToEntity.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> deleteRelatedEntity(id, entityId, cb)); + }); + newCfsRelatedToEntity.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> initRelatedEntity(id, entityId, ctx, cb)); + }); + } else { + callbackFor2.onSuccess(); + } + var oldProfileCfs = getCalculatedFieldsByEntityId(msg.getOldProfileId()); var newProfileCfs = getCalculatedFieldsByEntityId(msg.getProfileId()); var fieldsCount = oldProfileCfs.size() + newProfileCfs.size(); if (fieldsCount > 0) { - MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callbackFor2); var entityId = msg.getEntityId(); oldProfileCfs.forEach(ctx -> deleteCfForEntity(entityId, ctx.getCfId(), multiCallback)); newProfileCfs.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); } else { - callback.onSuccess(); + callbackFor2.onSuccess(); } } else if (msg.isOwnerChanged()) { onEntityOwnerChanged(msg, callback); + } else if (msg.isRelationChanged()) { + onRelationUpdated(msg, callback); } else { callback.onSuccess(); } } private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { - switch (msg.getEntityId().getEntityType()) { - case DEVICE, ASSET -> entityProfileCache.removeEntityId(msg.getEntityId()); - case CUSTOMER -> ownerEntities.remove(msg.getEntityId()); + if (msg.isRelationChanged()) { + onRelationDeleted(msg, callback); + } else { + switch (msg.getEntityId().getEntityType()) { + case DEVICE, ASSET -> entityProfileCache.removeEntityId(msg.getEntityId()); + case CUSTOMER -> ownerEntities.remove(msg.getEntityId()); + } + ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); + + getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getProfileId()).forEach(ctx -> { + applyToTargetCfEntityActors(ctx, callback, (id, cb) -> deleteRelatedEntity(id, msg.getEntityId(), cb)); + }); + if (isMyPartition(msg.getEntityId(), callback)) { + log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); + getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); + } } - ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); - if (isMyPartition(msg.getEntityId(), callback)) { - log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); - getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); + } + + private void onRelationUpdated(ComponentLifecycleMsg msg, TbCallback callback) { + try { + MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); + EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); + + EntityId toId = entityRelation.getTo(); + EntityId fromId = entityRelation.getFrom(); + String relationType = entityRelation.getType(); + EntityId toIdProfile = getProfileId(tenantId, toId); + EntityId fromIdProfile = getProfileId(tenantId, fromId); + + List toIdMatches = new ArrayList<>(); + List cfsByToId = getCalculatedFieldsByEntityId(toId); + List cfsByToProfileId = getCalculatedFieldsByEntityId(toIdProfile); + List cfsByToIdOrItsProfileId = new ArrayList<>(); + cfsByToIdOrItsProfileId.addAll(cfsByToId); + cfsByToIdOrItsProfileId.addAll(cfsByToProfileId); + + cfsByToIdOrItsProfileId.forEach(cf -> { + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(fromIdProfile)) { + toIdMatches.add(cf); + } + }); + + MultipleTbCallback toCfsCallback = new MultipleTbCallback(toIdMatches.size(), callbackForToAndFrom); + toIdMatches.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, toCfsCallback, (entityId, cb) -> initRelatedEntity(entityId, fromId, ctx, cb)); + }); + + List fromIdMatches = new ArrayList<>(); + List cfsByFromId = getCalculatedFieldsByEntityId(fromId); + List cfsByFromProfileId = getCalculatedFieldsByEntityId(fromIdProfile); + List cfsByFromIdOrItsProfileId = new ArrayList<>(); + cfsByFromIdOrItsProfileId.addAll(cfsByFromId); + cfsByFromIdOrItsProfileId.addAll(cfsByFromProfileId); + + cfsByFromIdOrItsProfileId.forEach(cf -> { + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(toIdProfile)) { + fromIdMatches.add(cf); + } + }); + + MultipleTbCallback fromCfsCallback = new MultipleTbCallback(fromIdMatches.size(), callbackForToAndFrom); + fromIdMatches.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, fromCfsCallback, (entityId, cb) -> initRelatedEntity(entityId, toId, ctx, cb)); + }); + + + } catch (Exception e) { + callback.onSuccess(); + } + } + + private void onRelationDeleted(ComponentLifecycleMsg msg, TbCallback callback) { + try { + MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); + EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); + + EntityId toId = entityRelation.getTo(); + EntityId fromId = entityRelation.getFrom(); + String relationType = entityRelation.getType(); + EntityId toIdProfile = getProfileId(tenantId, toId); + EntityId fromIdProfile = getProfileId(tenantId, fromId); + + List toIdMatches = new ArrayList<>(); + List cfsByToId = getCalculatedFieldsByEntityId(toId); + List cfsByToProfileId = getCalculatedFieldsByEntityId(toIdProfile); + List cfsByToIdOrItsProfileId = new ArrayList<>(); + cfsByToIdOrItsProfileId.addAll(cfsByToId); + cfsByToIdOrItsProfileId.addAll(cfsByToProfileId); + + cfsByToIdOrItsProfileId.forEach(cf -> { + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(fromIdProfile)) { + toIdMatches.add(cf); + } + }); + + MultipleTbCallback toCfsCallback = new MultipleTbCallback(toIdMatches.size(), callbackForToAndFrom); + toIdMatches.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, toCfsCallback, (entityId, cb) -> deleteRelatedEntity(entityId, fromId, cb)); + }); + + List fromIdMatches = new ArrayList<>(); + List cfsByFromId = getCalculatedFieldsByEntityId(fromId); + List cfsByFromProfileId = getCalculatedFieldsByEntityId(fromIdProfile); + List cfsByFromIdOrItsProfileId = new ArrayList<>(); + cfsByFromIdOrItsProfileId.addAll(cfsByFromId); + cfsByFromIdOrItsProfileId.addAll(cfsByFromProfileId); + + cfsByFromIdOrItsProfileId.forEach(cf -> { + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(toIdProfile)) { + fromIdMatches.add(cf); + } + }); + + MultipleTbCallback fromCfsCallback = new MultipleTbCallback(fromIdMatches.size(), callbackForToAndFrom); + fromIdMatches.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, fromCfsCallback, (entityId, cb) -> deleteRelatedEntity(entityId, toId, cb)); + }); + + + } catch (Exception e) { + callback.onSuccess(); } } @@ -302,6 +481,9 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } calculatedFields.put(cf.getId(), cfCtx); + if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + } // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); @@ -333,6 +515,9 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(newCfCtx).eventEntity(newCfCtx.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(newCf.getId(), newCfCtx); + if (newCf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(newCf.getId(), aggConfig.buildTrigger()); + } List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new CopyOnWriteArrayList<>(); boolean found = false; @@ -417,6 +602,106 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } else { callback.onSuccess(); } + // process all aggregation cfs (if any); + List aggregationCalculatedFields = filterAggregationCfs(msg); + if (!aggregationCalculatedFields.isEmpty()) { + cfExecService.pushMsgToLinks(msg, aggregationCalculatedFields, callback); + } else { + callback.onSuccess(); + } + } + + private List filterAggregationCfs(CalculatedFieldTelemetryMsg msg) { + EntityId entityId = msg.getEntityId(); + + List aggregationCalculatedFields = cfTriggers.entrySet().stream() + .filter(entry -> aggMatches(entry.getValue(), msg.getProto())) + .map(Entry::getKey) + .map(calculatedFields::get) + .filter(Objects::nonNull) + .toList(); + + List filteredByRelationCfs = new ArrayList<>(); + for (CalculatedFieldCtx cf : aggregationCalculatedFields) { + EntityId cfEntityId = cf.getEntityId(); + if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + RelationPathLevel relation = aggConfig.getSource().getRelation(); + EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) + ? cfEntityId + : getProfileId(tenantId, cfEntityId); + EntityId targetEntity = switch (relation.direction()) { + case FROM -> + relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).getFrom(); + case TO -> + relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0).getTo(); + }; + if (targetEntity != null) { + filteredByRelationCfs.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), targetEntity)); + } + } + } + return filteredByRelationCfs; + } + + private List getCalculatedFieldsRelatedToEntity(EntityId entityId, EntityId profileId) { + List aggCFsUsedProfile = cfTriggers.entrySet().stream() + .filter(entry -> entry.getValue().matchesProfile(profileId)) + .map(Entry::getKey) + .map(calculatedFields::get) + .filter(Objects::nonNull) + .toList(); + + List filteredByRelationCfs = new ArrayList<>(); + for (CalculatedFieldCtx cf : aggCFsUsedProfile) { + CalculatedFieldEntityCtxId calculatedFieldEntityCtxId = filterCfByRelationWithEntity(entityId, cf); + if (calculatedFieldEntityCtxId != null) { + filteredByRelationCfs.add(cf); + } + } + return filteredByRelationCfs; + } + + private CalculatedFieldEntityCtxId filterCfByRelationWithEntity(EntityId entityId, CalculatedFieldCtx cf) { + EntityId cfEntityId = cf.getEntityId(); + if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + RelationPathLevel relation = aggConfig.getSource().getRelation(); + EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) + ? cfEntityId + : getProfileId(tenantId, cfEntityId); + EntityId targetEntity = switch (relation.direction()) { + case FROM -> { + EntityRelation entityRelation = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); + yield entityRelation == null ? null : entityRelation.getFrom(); + } + case TO -> { + EntityRelation entityRelation = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0); + yield entityRelation == null ? null : entityRelation.getTo(); + } + }; + if (targetEntity != null) { + return new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), targetEntity); + } + } + return null; + } + + private boolean aggMatches(CfAggTrigger cfAggTrigger, CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTsDataList().isEmpty()) { + List updatedTelemetry = proto.getTsDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return cfAggTrigger.matchesTimeSeries(updatedTelemetry); + } else if (!proto.getAttrDataList().isEmpty()) { + AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); + List updatedTelemetry = proto.getAttrDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return cfAggTrigger.matchesAttributes(updatedTelemetry, scope); + } else if (!proto.getRemovedTsKeysList().isEmpty()) { + return cfAggTrigger.matchesTimeSeriesKeys(proto.getRemovedTsKeysList()); + } else { + return cfAggTrigger.matchesAttributesKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); + } } public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { @@ -536,6 +821,16 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware getOrCreateActor(entityId).tell(msg); } + private void deleteRelatedEntity(EntityId entityId, EntityId relatedEntityId, TbCallback callback) { + log.debug("Pushing delete related entity msg to specific actor [{}]", relatedEntityId); + getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, relatedEntityId, callback)); + } + + private void initRelatedEntity(EntityId entityId, EntityId relatedEntityId, CalculatedFieldCtx cf, TbCallback callback) { + log.debug("Pushing init related entity msg to specific actor [{}]", relatedEntityId); + getOrCreateActor(entityId).tell(new CalculatedFieldRelatedEntityMsg(tenantId, relatedEntityId, cf, callback)); + } + private void deleteCfForEntity(EntityId entityId, CalculatedFieldId cfId, TbCallback callback) { log.debug("Pushing delete CF msg to specific actor [{}]", entityId); getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, cfId, callback)); @@ -614,6 +909,9 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(cf.getId(), cfCtx); + if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + } // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java new file mode 100644 index 0000000000..d3bdd9cc5c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +@Data +public class CalculatedFieldRelatedEntityMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldCtx calculatedField; + private final TbCallback callback; + + public CalculatedFieldRelatedEntityMsg(TenantId tenantId, + EntityId entityId, + CalculatedFieldCtx calculatedField, + TbCallback callback) { + this.tenantId = tenantId; + this.entityId = entityId; + this.calculatedField = calculatedField; + this.callback = callback; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_RELATED_ENTITY_MSG; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index d17148a502..304b747f26 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -26,7 +26,10 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; @@ -36,6 +39,8 @@ import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.relation.RelationService; @@ -44,7 +49,9 @@ import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -98,6 +105,7 @@ public abstract class AbstractCalculatedFieldProcessingService { } yield futures; } + case LATEST_VALUES_AGGREGATION -> fetchAggregationArgumentFutures(ctx, entityId); }; return Futures.whenAllComplete(argFutures.values()) .call(() -> resolveArgumentFutures(argFutures), @@ -114,6 +122,19 @@ public abstract class AbstractCalculatedFieldProcessingService { return resolveOwnerArgument(tenantId, entityId); } + private List resolveRelatedEntities(TenantId tenantId, EntityId entityId, AggSource aggSource) { + RelationPathLevel relation = aggSource.getRelation(); + return switch (relation.direction()) { + case FROM -> aggSource.getEntityProfiles().stream() + .map(profile -> relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), profile)) + .flatMap(Collection::stream) + .map(EntityRelation::getTo) + .toList(); + case TO -> + aggSource.getEntityProfiles().stream().map(profile -> relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), profile).getFrom()).toList(); + }; + } + protected Map resolveArgumentFutures(Map> argFutures) { return argFutures.entrySet().stream() .collect(Collectors.toMap( @@ -176,6 +197,50 @@ public abstract class AbstractCalculatedFieldProcessingService { return ownerService.getOwner(tenantId, entityId); } + private Map> fetchAggregationArgumentFutures(CalculatedFieldCtx ctx, EntityId entityId) { + LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + + List entityIds = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getSource()); + + Map> futures = new HashMap<>(); + aggConfig.getInputs().forEach((key, refKey) -> { + Argument argument = new Argument(); + argument.setRefEntityKey(refKey); + futures.put(key, fetchAggArgumentEntry(ctx.getTenantId(), entityIds, argument, System.currentTimeMillis())); + }); + return futures; + } + + public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) { + List>> futures = aggEntities.stream() + .map(entityId -> fetchSingleAggArgumentEntry(tenantId, entityId, argument, startTs)) + .toList(); + + ListenableFuture>> allFutures = Futures.allAsList(futures); + + return Futures.transform(allFutures, + entries -> ArgumentEntry.createAggArgument( + entries.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) + ), + MoreExecutors.directExecutor()); + } + + protected ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { + CalculatedFieldConfiguration configuration = ctx.getCalculatedField().getConfiguration(); + LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) configuration; + Map> futures = new HashMap<>(); + aggConfig.getInputs().forEach((key, refKey) -> { + Argument argument = new Argument(); + argument.setRefEntityKey(refKey); + + ListenableFuture argumentEntryListenableFuture = fetchAggArgumentEntry(ctx.getTenantId(), List.of(entityId), argument, System.currentTimeMillis()); + futures.put(key, argumentEntryListenableFuture); + }); + return Futures.whenAllComplete(futures.values()) + .call(() -> resolveArgumentFutures(futures), + MoreExecutors.directExecutor()); + } + private ListenableFuture fetchGeofencingKvEntry(TenantId tenantId, List geofencingEntities, Argument argument) { if (argument.getRefEntityKey().getType() != ArgumentType.ATTRIBUTE) { throw new IllegalStateException("Unsupported argument key type: " + argument.getRefEntityKey().getType()); @@ -244,6 +309,38 @@ public abstract class AbstractCalculatedFieldProcessingService { }, calculatedFieldCallbackExecutor)); } + protected ListenableFuture> fetchSingleAggArgumentEntry(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { + return switch (argument.getRefEntityKey().getType()) { + case TS_ROLLING -> throw new IllegalStateException("TS_ROLLING is not supported for aggregation"); + case ATTRIBUTE -> fetchAttributeAggEntry(tenantId, entityId, argument, startTs); + case TS_LATEST -> fetchTsLatestAggEntry(tenantId, entityId, argument, startTs); + }; + } + + private ListenableFuture> fetchAttributeAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultLastUpdateTs) { + log.trace("[{}][{}] Fetching attribute for key {}", tenantId, entityId, argument.getRefEntityKey()); + var attributeOptFuture = attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()); + return Futures.transform(attributeOptFuture, attrOpt -> { + log.debug("[{}][{}] Fetched attribute for key {}: {}", tenantId, entityId, argument.getRefEntityKey(), attrOpt); + AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, 0L)); + AggSingleArgumentEntry entry = new AggSingleArgumentEntry(entityId, attributeKvEntry); + return Map.entry(entityId, entry); + }, calculatedFieldCallbackExecutor); + } + + protected ListenableFuture> fetchTsLatestAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultTs) { + String key = argument.getRefEntityKey().getKey(); + log.trace("[{}][{}] Fetching latest timeseries {}", tenantId, entityId, key); + return Futures.transform( + timeseriesService.findLatest(tenantId, entityId, key), + result -> { + log.debug("[{}][{}] Fetched latest timeseries {}: {}", tenantId, entityId, key, result); + Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))); + AggSingleArgumentEntry entry = new AggSingleArgumentEntry(entityId, tsKvEntry.get()); + return Map.entry(entityId, entry); + }, calculatedFieldCallbackExecutor); + } + private ReadTsKvQuery buildTsRollingQuery(TenantId tenantId, Argument argument, long startTs, long endTs) { long maxDataPoints = apiLimitService.getLimit( tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index cc77913f4b..e32ca42f9c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -38,6 +39,8 @@ public interface CalculatedFieldCache { List getCalculatedFieldCtxsByEntityId(EntityId entityId); + List getCalculatedFieldCtxsByTrigger(EntityId profileId, Predicate cfAggFilter); + boolean hasCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter); void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index a9139572b8..4b3e994f23 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -33,6 +33,8 @@ public interface CalculatedFieldProcessingService { ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId); + ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId); + Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId); Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 36210d7302..ae8238fb8a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; @@ -41,6 +43,8 @@ import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -48,6 +52,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Predicate; +import java.util.stream.Collectors; @Service @Slf4j @@ -68,6 +73,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + private final ConcurrentMap cfTriggers = new ConcurrentHashMap<>(); private final ConcurrentMap> ownerEntities = new ConcurrentHashMap<>(); @@ -81,6 +87,9 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { cfs.forEach(cf -> { if (cf != null) { calculatedFields.putIfAbsent(cf.getId(), cf); + if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + } } }); calculatedFields.values().forEach(cf -> { @@ -146,6 +155,16 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { .toList(); } + @Override + public List getCalculatedFieldCtxsByTrigger(EntityId profileId, Predicate cfAggFilter) { + return cfTriggers.entrySet().stream() + .filter(entry -> entry.getValue().matches(profileId, cfAggFilter)) + .map(Map.Entry::getKey) + .map(this::getCalculatedFieldCtx) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + @Override public boolean hasCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter) { List entityCfs = getCalculatedFieldCtxsByEntityId(entityId); @@ -155,6 +174,10 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } } + return hasCalculatedFieldsByProfile(tenantId, entityId, filter); + } + + public boolean hasCalculatedFieldsByProfile(TenantId tenantId, EntityId entityId, Predicate filter) { EntityId profileId = getProfileId(tenantId, entityId); if (profileId != null) { List profileCfs = getCalculatedFieldCtxsByEntityId(profileId); @@ -183,6 +206,9 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + if (configuration instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(calculatedField.getId(), aggConfig.buildTrigger()); + } calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); configuration.getReferencedEntities().stream() @@ -214,6 +240,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); entityIdCalculatedFieldLinks.forEach((entityId, calculatedFieldLinks) -> calculatedFieldLinks.removeIf(link -> link.getCalculatedFieldId().equals(calculatedFieldId))); log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); + cfTriggers.remove(calculatedFieldId); + log.debug("[{}] evict calculated field from cached triggers: {}", calculatedFieldId, oldCalculatedField); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index 9b2964a736..b7bd4d87fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.service.cf; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; @@ -54,6 +56,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; @@ -87,6 +90,11 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF return super.fetchArguments(ctx, entityId, System.currentTimeMillis()); } + @Override + public ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { + return super.fetchAggArguments(ctx, entityId, System.currentTimeMillis()); + } + @Override public Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId) { // only scheduledSupported CF instances supports dynamic arguments scheduled updates diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index a3e049d1e2..a75df1fb40 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -35,7 +37,10 @@ import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -71,6 +76,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS private final CalculatedFieldCache calculatedFieldCache; private final TbClusterService clusterService; + private final RelationService relationService; @Override public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { @@ -81,6 +87,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matches(entries), cf -> cf.linkMatches(entityId, entries), cf -> cf.dynamicSourceMatches(request.getEntries()), + cfTrigger -> cfTrigger.matchesTimeSeries(entries), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -99,6 +106,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matches(entries, scope), cf -> cf.linkMatches(entityId, entries, scope), cf -> cf.dynamicSourceMatches(request.getEntries(), request.getScope()), + cfTrigger -> cfTrigger.matchesAttributes(entries, scope), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -116,6 +124,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matchesKeys(result, scope), cf -> cf.linkMatchesAttrKeys(entityId, result, scope), cf -> cf.matchesDynamicSourceKeys(result, request.getScope()), + cfTrigger -> cfTrigger.matchesAttributesKeys(result, scope), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -127,6 +136,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matchesKeys(result), cf -> cf.linkMatchesTsKeys(entityId, result), cf -> cf.matchesDynamicSourceKeys(result), + cfTrigger -> cfTrigger.matchesTimeSeriesKeys(result), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -134,11 +144,12 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS Predicate mainEntityFilter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter, + Predicate cfAggKeysFilter, Supplier msg, FutureCallback callback) { if (EntityType.TENANT.equals(entityId.getEntityType())) { tenantId = (TenantId) entityId; } - boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter, dynamicSourceFilter); + boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter, dynamicSourceFilter, cfAggKeysFilter); if (send) { ToCalculatedFieldMsg calculatedFieldMsg = msg.get(); clusterService.pushMsgToCalculatedFields(tenantId, entityId, calculatedFieldMsg, wrap(callback)); @@ -149,7 +160,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } } - private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter) { + private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter, Predicate cfAggKeysFilter) { if (!CalculatedField.SUPPORTED_REFERENCED_ENTITIES.contains(entityId.getEntityType())) { return false; } @@ -176,6 +187,26 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } } + List cfCtxs = calculatedFieldCache.getCalculatedFieldCtxsByTrigger(calculatedFieldCache.getProfileId(tenantId, entityId), cfAggKeysFilter); + for (CalculatedFieldCtx cfCtx : cfCtxs) { + EntityId cfEntityId = cfCtx.getEntityId(); + if (cfCtx.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + RelationPathLevel relation = aggConfig.getSource().getRelation(); + EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) + ? cfEntityId + : calculatedFieldCache.getProfileId(tenantId, cfEntityId); + EntityRelation entityRelation = switch (relation.direction()) { + case FROM -> + relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); + case TO -> + relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0); + }; + if (entityRelation != null) { + return true; + } + } + } + return false; } @@ -266,6 +297,10 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS return telemetryMsg; } + private boolean isProfileEntity(EntityType entityType) { + return EntityType.DEVICE_PROFILE.equals(entityType) || EntityType.ASSET_PROFILE.equals(entityType); + } + private static TbQueueCallback wrap(FutureCallback callback) { if (callback != null) { return new FutureCallbackWrapper(callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 2d43883131..f22b10a9c1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -22,6 +22,8 @@ import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import java.util.List; @@ -35,7 +37,9 @@ import java.util.Map; @JsonSubTypes({ @JsonSubTypes.Type(value = SingleValueArgumentEntry.class, name = "SINGLE_VALUE"), @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING"), - @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING") + @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING"), + @JsonSubTypes.Type(value = AggArgumentEntry.class, name = "AGGREGATE_LATEST"), + @JsonSubTypes.Type(value = AggSingleArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") }) public interface ArgumentEntry { @@ -66,4 +70,8 @@ public interface ArgumentEntry { return new GeofencingArgumentEntry(entityIdkvEntryMap); } + static ArgumentEntry createAggArgument(Map entityIdkvEntryMap) { + return new AggArgumentEntry(entityIdkvEntryMap, false); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java index 876bfa2a3f..30b92a78d3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java @@ -16,5 +16,5 @@ package org.thingsboard.server.service.cf.ctx.state; public enum ArgumentEntryType { - SINGLE_VALUE, TS_ROLLING, GEOFENCING + SINGLE_VALUE, TS_ROLLING, GEOFENCING, AGGREGATE_LATEST, AGGREGATE_LATEST_SINGLE } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index e442964280..f75711a107 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -123,7 +123,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, protected void validateNewEntry(String key, ArgumentEntry newEntry) {} - private void updateLastUpdateTimestamp(ArgumentEntry entry) { + protected void updateLastUpdateTimestamp(ArgumentEntry entry) { long newTs = this.latestTimestamp; if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { newTs = singleValueArgumentEntry.getTs(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index e08abb8b50..963ec3f06f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -42,6 +42,8 @@ import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.ScheduledUpdateSupportedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -81,6 +83,8 @@ public class CalculatedFieldCtx { private final Map mainEntityArguments; private final Map> linkedEntityArguments; private final Map dynamicEntityArguments; + private final List aggInputs; + private final Map aggregationInputs; private final List argNames; private Output output; private String expression; @@ -122,6 +126,8 @@ public class CalculatedFieldCtx { this.argNames = new ArrayList<>(); this.mainEntityGeofencingArgumentNames = new ArrayList<>(); this.linkedEntityAndCurrentOwnerGeofencingArgumentNames = new ArrayList<>(); + this.aggInputs = new ArrayList<>(); + this.aggregationInputs = new HashMap<>(); this.output = calculatedField.getConfiguration().getOutput(); if (calculatedField.getConfiguration() instanceof ArgumentsBasedCalculatedFieldConfiguration argBasedConfig) { this.arguments.putAll(argBasedConfig.getArguments()); @@ -165,6 +171,12 @@ public class CalculatedFieldCtx { this.scheduledUpdateIntervalMillis = scheduledConfig.isScheduledUpdateEnabled() ? TimeUnit.SECONDS.toMillis(scheduledConfig.getScheduledUpdateInterval()) : -1L; } this.requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); + if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + aggInputs.addAll(aggConfig.getInputs().values()); + aggregationInputs.putAll(aggConfig.getInputs()); + this.argNames.addAll(aggConfig.getInputs().keySet()); + this.scheduledUpdateIntervalMillis = aggConfig.getDeduplicationIntervalMillis(); + } this.systemContext = systemContext; this.tbelInvokeService = systemContext.getTbelInvokeService(); this.relationService = systemContext.getRelationService(); @@ -199,6 +211,19 @@ public class CalculatedFieldCtx { }); initialized = true; } + case LATEST_VALUES_AGGREGATION -> { + LatestValuesAggregationCalculatedFieldConfiguration configuration = (LatestValuesAggregationCalculatedFieldConfiguration) calculatedField.getConfiguration(); + configuration.getMetrics().forEach((key, metric) -> { + if (metric.getInput() instanceof AggFunctionInput functionInput) { + initTbelExpression(functionInput.getFunction()); + } + String filter = metric.getFilter(); + if (filter != null && !filter.isEmpty()) { + initTbelExpression(filter); + } + }); + initialized = true; + } } } @@ -242,6 +267,24 @@ public class CalculatedFieldCtx { return expression.executeScriptAsync(args.toArray()); } + public ListenableFuture evaluateTbelExpression(String expression, Map entries, long latestTimestamp) { + Map arguments = new LinkedHashMap<>(); + List args = new ArrayList<>(argNames.size() + 1); + args.add(new Object()); // first element is a ctx, but we will set it later; + for (String argName : argNames) { + var arg = entries.get(argName).toTbelCfArg(); + arguments.put(argName, arg); + if (arg instanceof TbelCfSingleValueArg svArg) { + args.add(svArg.getValue()); + } else { + args.add(arg); + } + } + args.set(0, new TbelCfCtx(arguments, latestTimestamp)); + + return tbelExpressions.get(expression).executeScriptAsync(args.toArray()); + } + public ScheduledFuture scheduleReevaluation(long delayMs, TbActorRef actorCtx) { log.debug("[{}] Scheduling CF reevaluation in {} ms", cfId, delayMs); // TODO: use single lazy-loaded instance of CalculatedFieldReevaluateMsg @@ -451,6 +494,25 @@ public class CalculatedFieldCtx { } } + public boolean aggMatches(CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTsDataList().isEmpty()) { + List updatedTelemetry = proto.getTsDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return matchesAggTimeSeries(updatedTelemetry); + } else if (!proto.getAttrDataList().isEmpty()) { + AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); + List updatedTelemetry = proto.getAttrDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return matchesAggAttributes(updatedTelemetry, scope); + } else if (!proto.getRemovedTsKeysList().isEmpty()) { + return matchesAggKeys(proto.getRemovedTsKeysList()); + } else { + return matchesAggAttributesKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); + } + } + public boolean linkMatches(EntityId entityId, CalculatedFieldTelemetryMsgProto proto) { if (!proto.getTsDataList().isEmpty()) { List updatedTelemetry = proto.getTsDataList().stream() @@ -482,6 +544,67 @@ public class CalculatedFieldCtx { return argNames; } + public boolean matchesAggKeys(List values) { + if (aggInputs.isEmpty() || values.isEmpty()) { + return false; + } + + for (String key : values) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); + if (aggInputs.contains(latestKey)) { + return true; + } + } + + return false; + } + + public boolean matchesAggTimeSeries(List values) { + if (aggInputs.isEmpty() || values.isEmpty()) { + return false; + } + + for (TsKvEntry tsKvEntry : values) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKvEntry.getKey(), ArgumentType.TS_LATEST, null); + if (aggInputs.contains(latestKey)) { + return true; + } + } + + return false; + } + + public boolean matchesAggAttributesKeys(List keys, AttributeScope scope) { + if (keys == null || keys.isEmpty()) { + return false; + } + + for (String key : keys) { + ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); + if (aggInputs.contains(attrKey)) { + return true; + } + } + + return false; + } + + public boolean matchesAggAttributes(List keys, AttributeScope scope) { + if (keys == null || keys.isEmpty()) { + return false; + } + + for (AttributeKvEntry attributeKvEntry : keys) { + ReferencedEntityKey attrKey = new ReferencedEntityKey(attributeKvEntry.getKey(), ArgumentType.ATTRIBUTE, scope); + if (aggInputs.contains(attrKey)) { + return true; + } + } + + return false; + } + + public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); } @@ -499,6 +622,11 @@ public class CalculatedFieldCtx { if (scheduledUpdateIntervalMillis != other.scheduledUpdateIntervalMillis) { return true; } + if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig + && other.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig + && !thisConfig.getMetrics().equals(otherConfig.getMetrics())) { + return true; + } return false; } @@ -517,6 +645,9 @@ public class CalculatedFieldCtx { if (hasGeofencingZoneGroupConfigurationChanges(other)) { return true; } + if (hasLatestValuesAggregationConfigurationChanges(other)) { + return true; + } return false; } @@ -528,6 +659,15 @@ public class CalculatedFieldCtx { return false; } + private boolean hasLatestValuesAggregationConfigurationChanges(CalculatedFieldCtx other) { + if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig + && other.calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig) { + return !thisConfig.getInputs().equals(otherConfig.getInputs()) || !thisConfig.getSource().equals(otherConfig.getSource()); + } + return false; + } + + public boolean hasRelationQueryDynamicArguments() { return relationQueryDynamicArguments && scheduledUpdateIntervalMillis != -1; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index cc7188e7a5..bd40ab7a05 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -32,6 +32,7 @@ import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculat import java.io.Closeable; import java.util.Map; +import java.util.concurrent.ExecutionException; import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArgumentProto; @@ -53,6 +54,8 @@ public interface CalculatedFieldState extends Closeable { long getLatestTimestamp(); + CalculatedFieldCtx getCtx(); + void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx); void init(); @@ -61,7 +64,7 @@ public interface CalculatedFieldState extends Closeable { void reset(); - ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx); + ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception; @JsonIgnore boolean isReady(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 5c1ed32e1d..288b486e83 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -37,11 +37,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; @AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { - private long ts; - private BasicKvEntry kvEntryValue; - private Long version; + protected long ts; + protected BasicKvEntry kvEntryValue; + protected Long version; - private boolean forceResetPrevious; + protected boolean forceResetPrevious; public static final Long DEFAULT_VERSION = -1L; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java new file mode 100644 index 0000000000..138053793f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; + +import java.util.Map; + +@Data +@AllArgsConstructor +public class AggArgumentEntry implements ArgumentEntry { + + private final Map aggInputs; + + private boolean forceResetPrevious; + + @Override + public ArgumentEntryType getType() { + return ArgumentEntryType.AGGREGATE_LATEST; + } + + @Override + public Object getValue() { + return aggInputs; + } + + @Override + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof AggArgumentEntry aggArgumentEntry) { + aggInputs.putAll(aggArgumentEntry.aggInputs); + return true; + } else if (entry instanceof AggSingleArgumentEntry aggSingleArgumentEntry) { + if (aggSingleArgumentEntry.isDeleted()) { + aggInputs.remove(aggSingleArgumentEntry.getEntityId()); + } else { + aggInputs.put(aggSingleArgumentEntry.getEntityId(), aggSingleArgumentEntry); + } + return true; + } else { + throw new IllegalArgumentException("Unsupported argument entry type for aggregation argument entry: " + entry.getType()); + } + } + + @Override + public boolean isEmpty() { + return aggInputs.isEmpty(); + } + + @Override + public TbelCfArg toTbelCfArg() { + return null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java new file mode 100644 index 0000000000..6b81d5380c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java @@ -0,0 +1,83 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class AggSingleArgumentEntry extends SingleValueArgumentEntry { + + private EntityId entityId; + private boolean deleted; + + public AggSingleArgumentEntry(EntityId entityId, TsKvProto entry) { + super(entry); + this.entityId = entityId; + } + + public AggSingleArgumentEntry(EntityId entityId, AttributeValueProto entry) { + super(entry); + this.entityId = entityId; + } + + public AggSingleArgumentEntry(EntityId entityId, KvEntry entry) { + super(entry); + this.entityId = entityId; + } + + public AggSingleArgumentEntry(EntityId entityId, long ts, BasicKvEntry kvEntryValue, Long version) { + super(ts, kvEntryValue, version); + this.entityId = entityId; + } + + @Override + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof AggSingleArgumentEntry singleValueEntry) { + if (singleValueEntry.getTs() <= ts) { + return false; + } + + Long newVersion = singleValueEntry.getVersion(); + if (newVersion == null || this.version == null || newVersion > this.version) { + this.ts = singleValueEntry.getTs(); + this.version = newVersion; + this.kvEntryValue = singleValueEntry.getKvEntryValue(); + this.entityId = singleValueEntry.getEntityId(); + return true; + } + } else { + throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); + } + return false; + } + + @Override + public ArgumentEntryType getType() { + return ArgumentEntryType.AGGREGATE_LATEST_SINGLE; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java new file mode 100644 index 0000000000..d25cde020c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java @@ -0,0 +1,161 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.actors.TbActorRef; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.BaseCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.aggregation.function.AggEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.function.AggFunctionFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +@Slf4j +@Data +public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedFieldState { + + private long lastArgsRefreshTs = -1; + private long lastMetricsEvalTs = -1; + private long deduplicationInterval = -1; + private Map metrics; + + private final Map> inputs = new HashMap<>(); + + public LatestValuesAggregationCalculatedFieldState(EntityId entityId) { + super(entityId); + } + + @Override + public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) { + super.setCtx(ctx, actorCtx); + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + metrics = configuration.getMetrics(); + deduplicationInterval = configuration.getDeduplicationIntervalMillis(); + } + + @Override + public void reset() { // must reset everything dependent on arguments + super.reset(); + lastArgsRefreshTs = -1; + lastMetricsEvalTs = -1; + metrics = null; + } + + @Override + public void init() { + super.init(); +// long scheduledUpdateIntervalMillis = ctx.getScheduledUpdateIntervalMillis(); +// ctx.scheduleReevaluation(scheduledUpdateIntervalMillis, actorCtx); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.LATEST_VALUES_AGGREGATION; + } + + @Override + public Map update(Map argumentValues, CalculatedFieldCtx ctx) { + Map updatedArguments = super.update(argumentValues, ctx); + lastArgsRefreshTs = System.currentTimeMillis(); + for (Map.Entry argEntry : arguments.entrySet()) { + String key = argEntry.getKey(); + AggArgumentEntry aggArgumentEntry = (AggArgumentEntry) argEntry.getValue(); + Map aggInputs = aggArgumentEntry.getAggInputs(); + aggInputs.forEach((entityId, argumentEntry) -> { + inputs.computeIfAbsent(entityId, k -> new HashMap<>()).put(key, argumentEntry); + }); + } + return updatedArguments; + } + + @Override + public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception { + boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - deduplicationInterval; + boolean argsUpdatedDuringInterval = lastArgsRefreshTs > lastMetricsEvalTs; + if (intervalPassed && argsUpdatedDuringInterval) { + ObjectNode aggResult = JacksonUtil.newObjectNode(); + for (Entry entry : metrics.entrySet()) { + String metricKey = entry.getKey(); + AggMetric metric = entry.getValue(); + + AggEntry aggMetric = AggFunctionFactory.createAggFunction(metric.getFunction()); + + for (Map entityInputs : inputs.values()) { + if (applyAggregation(metric.getFilter(), entityInputs)) { + Object arg = resolveAggregationInput(metric.getInput(), entityInputs); + if (arg != null) { + aggMetric.update(arg); + } + } + } + + aggMetric.result().ifPresent(result -> { + aggResult.set(metricKey, JacksonUtil.valueToTree(result)); + }); + } + Output output = ctx.getOutput(); + lastMetricsEvalTs = System.currentTimeMillis(); + ctx.scheduleReevaluation(deduplicationInterval, actorCtx); + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .type(output.getType()) + .scope(output.getScope()) + .result(aggResult) + .build()); + } else { + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .result(null) + .build()); + } + } + + private boolean applyAggregation(String filter, Map entityInputs) throws Exception { + if (filter == null || filter.isEmpty()) { + return true; + } else { + Object filterResult = ctx.evaluateTbelExpression(filter, entityInputs, getLatestTimestamp()).get(); + return filterResult instanceof Boolean booleanResult && booleanResult; + } + } + + private Object resolveAggregationInput(AggInput aggInput, Map entityInputs) throws Exception { + if (aggInput instanceof AggFunctionInput functionInput) { + return ctx.evaluateTbelExpression(functionInput.getFunction(), entityInputs, getLatestTimestamp()).get(); + } else { + String inputKey = ((AggKeyInput) aggInput).getKey(); + return entityInputs.get(inputKey).getValue(); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json new file mode 100644 index 0000000000..d402f23c6e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json @@ -0,0 +1,65 @@ +{ + "type": "LATEST_VALUES_AGGREGATION", + "name": "Occupied spaces", + "debugSettings": { + "failuresEnabled": true, + "allEnabled": true, + "allEnabledUntil": 1769907492297 + }, + "entityId": { + "entityType": "ASSET", + "id": "cc830710-a4cf-11f0-87cb-2d6683c4fccf" + }, + "configuration": { + "type": "LATEST_VALUES_AGGREGATION", + "source": { + "relation": { + "direction": "FROM", + "relationType": "Contains" + }, + "entityProfiles": [ + { + "entityType": "DEVICE_PROFILE", + "id": "d7a05580-a4cf-11f0-87cb-2d6683c4fccf" + } + ] + }, + "inputs": { + "oc": { + "key": "occupied", + "type": "TS_LATEST" + } + }, + "deduplicationIntervalMillis": 10000, + "metrics": { + "totalSpaces": { + "function": "COUNT", + "input": { + "type": "function", + "function" : "return 1;" + } + }, + "occupiedSpaces": { + "function": "COUNT", + "filter": "return oc == true", + "input": { + "type": "key", + "key" : "oc" + } + }, + "freeSpaces": { + "function": "COUNT", + "filter": "return oc == false", + "input": { + "type": "key", + "key" : "oc" + } + } + }, + "output": { + "type": "TIME_SERIES", + "decimals": 2 + }, + "useLatestTsFromInputs": "true" + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java new file mode 100644 index 0000000000..4239e94fec --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.util.Optional; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = AvgAggEntry.class, name = "AVG"), + @JsonSubTypes.Type(value = CountAggEntry.class, name = "COUNT"), + @JsonSubTypes.Type(value = CountUniqueAggEntry.class, name = "COUNT_UNIQUE"), + @JsonSubTypes.Type(value = MaxAggEntry.class, name = "MAX"), + @JsonSubTypes.Type(value = MinAggEntry.class, name = "MIN"), + @JsonSubTypes.Type(value = SumAggEntry.class, name = "SUM") +}) +public interface AggEntry { + + AggFunction getType(); + + void update(Object value); + + Optional result(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java new file mode 100644 index 0000000000..5ccc355b1f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +public class AggFunctionFactory { + + public static AggEntry createAggFunction(AggFunction function) { + return switch (function) { + case MIN -> new MinAggEntry(); + case MAX -> new MaxAggEntry(); + case SUM -> new SumAggEntry(); + case AVG -> new AvgAggEntry(); + case COUNT -> new CountAggEntry(); + case COUNT_UNIQUE -> new CountUniqueAggEntry(); + }; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java new file mode 100644 index 0000000000..ad1f2ee8a8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.math.BigDecimal; +import java.math.RoundingMode; + +public class AvgAggEntry extends BaseAggEntry { + + private BigDecimal sum = BigDecimal.ZERO; + private long count = 0L; + + @Override + protected void doUpdate(double value) { + if (value != 0.0) { + sum = sum.add(BigDecimal.valueOf(value)); + } + this.count++; + } + + @Override + protected double prepareResult() { + return sum.divide(BigDecimal.valueOf(count), 2, RoundingMode.HALF_UP).doubleValue(); + } + + @Override + public AggFunction getType() { + return AggFunction.AVG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java new file mode 100644 index 0000000000..0c12bd13c0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import java.util.Optional; + +public abstract class BaseAggEntry implements AggEntry { + + private boolean hasResult = false; + + @Override + public void update(Object value) { + doUpdate(extractDoubleValue(value)); + hasResult = true; + } + + @Override + public Optional result() { + if (hasResult) { + hasResult = false; + return Optional.of(prepareResult()); + } else { + return Optional.empty(); + } + } + + protected abstract void doUpdate(double value); + + protected abstract double prepareResult(); + + protected double extractDoubleValue(Object value) { + try { + return Double.parseDouble(value.toString()); + } catch (Exception e) { + throw new NumberFormatException("Cannot parse value " + value.toString()); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java new file mode 100644 index 0000000000..469048d62d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.util.Optional; + +public class CountAggEntry implements AggEntry { + + private long count = 0L; + + @Override + public void update(Object value) { + count++; + } + + @Override + public Optional result() { + return Optional.of(count); + } + + @Override + public AggFunction getType() { + return AggFunction.COUNT; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java new file mode 100644 index 0000000000..b8b0b92470 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +public class CountUniqueAggEntry implements AggEntry { + + private Set items; + + @Override + public void update(Object value) { + if (value != null) { + items.add(JacksonUtil.toString(value)); + } + } + + @Override + public Optional result() { + return Optional.of(items.size()); + } + + @Override + public AggFunction getType() { + return AggFunction.COUNT_UNIQUE; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java new file mode 100644 index 0000000000..6e4235b72f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +public class MaxAggEntry extends BaseAggEntry { + + private double max = -Double.MAX_VALUE; + + @Override + protected void doUpdate(double value) { + if (value > max) { + max = value; + } + } + + @Override + protected double prepareResult() { + return max; + } + + @Override + public AggFunction getType() { + return AggFunction.MAX; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java new file mode 100644 index 0000000000..eeacc3a7d9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +public class MinAggEntry extends BaseAggEntry { + + private double min = Double.MAX_VALUE; + + @Override + protected void doUpdate(double value) { + if (value < min) { + min = value; + } + } + + @Override + protected double prepareResult() { + return min; + } + + @Override + public AggFunction getType() { + return AggFunction.MIN; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java new file mode 100644 index 0000000000..b90817d784 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.math.BigDecimal; + +public class SumAggEntry extends BaseAggEntry { + + private BigDecimal sum = BigDecimal.ZERO; + + @Override + protected void doUpdate(double value) { + if (value != 0.0) { + sum = sum.add(BigDecimal.valueOf(value)); + } + } + + @Override + protected double prepareResult() { + return sum.doubleValue(); + } + + @Override + public AggFunction getType() { + return AggFunction.SUM; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index b9fd38f1e2..f762cf5be5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; @@ -61,6 +62,7 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeSynchronizationManager; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; +import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.gen.transport.TransportProtos.EntityActionEventProto; @@ -271,6 +273,15 @@ public class EntityStateSourcingListener { } } + @TransactionalEventListener(fallbackExecution = true) + public void handleEvent(RelationActionEvent relationEvent) { + if (relationEvent.getActionType() == ActionType.RELATION_ADD_OR_UPDATE) { + tbClusterService.onRelationUpdated(relationEvent.getTenantId(), relationEvent.getRelation(), TbQueueCallback.EMPTY); + } else if (relationEvent.getActionType() == ActionType.RELATION_DELETED) { + tbClusterService.onRelationDeleted(relationEvent.getTenantId(), relationEvent.getRelation(), TbQueueCallback.EMPTY); + } + } + private void onTenantUpdate(Tenant tenant, ComponentLifecycleEvent lifecycleEvent) { tbClusterService.onTenantChange(tenant, null); tbClusterService.broadcastEntityStateChangeEvent(tenant.getId(), tenant.getId(), lifecycleEvent); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index f1494658a8..22faeaf61b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -56,6 +56,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; @@ -369,6 +370,17 @@ public class DefaultTbClusterService implements TbClusterService { broadcast(new ComponentLifecycleMsg(tenantId, entityId, state)); } + @Override + public void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, EntityId profileId, ComponentLifecycleEvent state) { + log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); + broadcast(ComponentLifecycleMsg.builder() + .tenantId(tenantId) + .entityId(entityId) + .profileId(profileId) + .event(state) + .build()); + } + @Override public void onDeviceProfileChange(DeviceProfile deviceProfile, DeviceProfile oldDeviceProfile, TbQueueCallback callback) { boolean isFirmwareChanged = false; @@ -420,13 +432,13 @@ public class DefaultTbClusterService implements TbClusterService { gatewayNotificationsService.onDeviceDeleted(device); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); - broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); + broadcastEntityStateChangeEvent(tenantId, deviceId, device.getDeviceProfileId(), ComponentLifecycleEvent.DELETED); } @Override public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { AssetId assetId = asset.getId(); - broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); + broadcastEntityStateChangeEvent(tenantId, assetId, asset.getAssetProfileId(), ComponentLifecycleEvent.DELETED); } @Override @@ -723,6 +735,30 @@ public class DefaultTbClusterService implements TbClusterService { broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED); } + @Override + public void onRelationUpdated(TenantId tenantId, EntityRelation entityRelation, TbQueueCallback callback) { + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(tenantId) + .entityId(entityRelation.getFrom()) + .relationChanged(true) + .event(ComponentLifecycleEvent.UPDATED) + .info(JacksonUtil.valueToTree(entityRelation)) + .build(); + broadcast(msg); + } + + @Override + public void onRelationDeleted(TenantId tenantId, EntityRelation entityRelation, TbQueueCallback callback) { + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(tenantId) + .entityId(entityRelation.getFrom()) + .relationChanged(true) + .event(ComponentLifecycleEvent.DELETED) + .info(JacksonUtil.valueToTree(entityRelation)) + .build(); + broadcast(msg); + } + @Override public void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId originatorEdgeId) { if (!edgesEnabled) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 7382ef1c4d..761b69024f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -184,24 +184,34 @@ public abstract class AbstractConsumerService new ScriptCalculatedFieldState(entityId); case GEOFENCING -> new GeofencingCalculatedFieldState(entityId); case ALARM -> new AlarmCalculatedFieldState(entityId); + case LATEST_VALUES_AGGREGATION -> new LatestValuesAggregationCalculatedFieldState(entityId); }; } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 4319f72448..d4ee7bb2e3 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.util.KvProtoUtil; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.AggSingleArgumentEntryProto; import org.thingsboard.server.gen.transport.TransportProtos.AlarmRuleStateProto; import org.thingsboard.server.gen.transport.TransportProtos.AlarmStateProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; @@ -45,12 +46,16 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingZoneState; +import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.TreeMap; @@ -92,7 +97,10 @@ public class CalculatedFieldUtils { .setType(state.getType().name()); state.getArguments().forEach((argName, argEntry) -> { - if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + if (argEntry instanceof AggArgumentEntry aggArgumentEntry) { + aggArgumentEntry.getAggInputs() + .forEach((entityId, entry) -> builder.addAggArguments(toAggSingleArgumentProto(argName, entityId, entry))); + } else if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { builder.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); @@ -130,6 +138,17 @@ public class CalculatedFieldUtils { return ruleState; } + public static AggSingleArgumentEntryProto toAggSingleArgumentProto(String argName, EntityId entityId, ArgumentEntry argumentEntry) { + AggSingleArgumentEntryProto.Builder builder = AggSingleArgumentEntryProto.newBuilder() + .setEntityId(ProtoUtils.toProto(entityId)); + + if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + builder.setValue(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); + } + + return builder.build(); + } + public static SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() .setArgName(argName); @@ -187,6 +206,7 @@ public class CalculatedFieldUtils { case SCRIPT -> new ScriptCalculatedFieldState(id.entityId()); case GEOFENCING -> new GeofencingCalculatedFieldState(id.entityId()); case ALARM -> new AlarmCalculatedFieldState(id.entityId()); + case LATEST_VALUES_AGGREGATION -> new LatestValuesAggregationCalculatedFieldState(id.entityId()); }; proto.getSingleValueArgumentsList().forEach(argProto -> @@ -212,11 +232,37 @@ public class CalculatedFieldUtils { alarmState.setClearRuleState(fromAlarmRuleStateProto(alarmStateProto.getClearRuleState(), alarmState)); } } + case LATEST_VALUES_AGGREGATION -> { + LatestValuesAggregationCalculatedFieldState aggState = (LatestValuesAggregationCalculatedFieldState) state; + Map> arguments = new HashMap<>(); + proto.getAggArgumentsList().forEach(argProto -> { + AggSingleArgumentEntry entry = fromAggSingleValueArgumentProto(argProto); + arguments.computeIfAbsent(argProto.getValue().getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); + }); + arguments.forEach((argName, entityInputs) -> { + aggState.getArguments().put(argName, new AggArgumentEntry(entityInputs, false)); + }); + } } return state; } + public static AggSingleArgumentEntry fromAggSingleValueArgumentProto(AggSingleArgumentEntryProto proto) { + if (!proto.hasValue()) { + return new AggSingleArgumentEntry(); + } + EntityId entityId = ProtoUtils.fromProto(proto.getEntityId()); + SingleValueArgumentProto singleValueArgument = proto.getValue(); + TsValueProto tsValueProto = singleValueArgument.getValue(); + return new AggSingleArgumentEntry( + entityId, + tsValueProto.getTs(), + (BasicKvEntry) KvProtoUtil.fromTsValueProto(singleValueArgument.getArgName(), tsValueProto), + singleValueArgument.getVersion() + ); + } + public static SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { if (!proto.hasValue()) { return new SingleValueArgumentEntry(); diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java new file mode 100644 index 0000000000..1a5fc2e3d3 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -0,0 +1,314 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cf; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; +import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.data.DeviceData; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationPathLevel; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.thingsboard.server.cf.CalculatedFieldIntegrationTest.POLL_INTERVAL; + +@DaoSqlTest +public class LatestValuesAggregationCalculatedFieldTest extends AbstractControllerTest { + + private Tenant savedTenant; + + private DeviceProfile deviceProfile; + private Device device1; + private String accessToken1 = "1234567890111"; + private Device device2; + private String accessToken2 = "1234567890222"; + + private AssetProfile assetProfile; + private Asset asset; + + private CalculatedField calculatedField; + + private long deduplicationInterval = 10000; + + @Before + public void beforeTest() throws Exception { + loginSysAdmin(); + + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + savedTenant = saveTenant(tenant); + assertThat(savedTenant).isNotNull(); + + User tenantAdmin = new User(); + tenantAdmin.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin.setTenantId(savedTenant.getId()); + tenantAdmin.setEmail("tenant@thingsboard.org"); + tenantAdmin.setFirstName("John"); + tenantAdmin.setLastName("Doe"); + + createUserAndLogin(tenantAdmin, "testPassword"); + + deviceProfile = doPost("/api/deviceProfile", createDeviceProfile("Device Profile"), DeviceProfile.class); + device1 = createDevice("Device 1", deviceProfile.getId(), accessToken1); + device2 = createDevice("Device 2", deviceProfile.getId(), accessToken2); + + postTelemetry(device1.getId(), "{\"occupied\":true}"); + postTelemetry(device2.getId(), "{\"occupied\":false}"); + + assetProfile = doPost("/api/assetProfile", createAssetProfile("Asset Profile"), AssetProfile.class); + asset = createAsset("Asset", assetProfile.getId()); + + createEntityRelation(asset.getId(), device1.getId(), "Contains"); + createEntityRelation(asset.getId(), device2.getId(), "Contains"); + + calculatedField = createOccupancyCF(asset.getId(), List.of(deviceProfile.getId())); + + checkInitialCalculation(); + } + + @After + public void afterTest() throws Exception { + loginSysAdmin(); + + deleteTenant(savedTenant.getId()); + } + + @Test + public void testUpdateTelemetry_checkMetricsCalculation() throws Exception { + postTelemetry(device1.getId(), "{\"occupied\":false}"); + + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + }); + } + + @Test + public void testUpdateTelemetry_checkMetricsCalculationNotExecutedUntilDeduplicationInterval() throws Exception { + postTelemetry(device1.getId(), "{\"occupied\":false}"); + + await().alias("update telemetry -> no changes").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(this::checkInitialCalculationValues); + + postTelemetry(device2.getId(), "{\"occupied\":false}"); + + await().alias("create CF and perform initial calculation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + }); + } + + @Test + public void testChangeProfile_checkMetricsCalculation() throws Exception { + DeviceProfile deviceProfile2 = doPost("/api/deviceProfile", createDeviceProfile("Device Profile 2"), DeviceProfile.class); + device1.setDeviceProfileId(deviceProfile2.getId()); + device1 = doPost("/api/device?accessToken=" + accessToken1, device1, Device.class); + + postTelemetry(device1.getId(), "{\"occupied\":false}"); + + await().alias("change profile and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); + }); + } + + @Test + public void testAddEntityToProfile_checkMetricsCalculation() throws Exception { + Device device3 = createDevice("Device 3", deviceProfile.getId(), "1234567890333"); + + postTelemetry(device3.getId(), "{\"occupied\":true}"); + + await().alias("add entity to profile and no calculation (there is no relation between device and asset)").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(this::checkInitialCalculationValues); + + createEntityRelation(asset.getId(), device3.getId(), "Contains"); + + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); + }); + } + + @Test + public void testDeleteRelation_checkMetricsCalculation() throws Exception { + deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); + + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); + }); + } + + private void checkInitialCalculation() { + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(this::checkInitialCalculationValues); + } + + private void checkInitialCalculationValues() throws Exception { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + } + + private CalculatedField createOccupancyCF(EntityId entityId, List profiles) { + Map aggMetrics = new HashMap<>(); + + AggMetric freeSpaces = new AggMetric(); + freeSpaces.setFunction(AggFunction.COUNT); + freeSpaces.setFilter("return oc == false"); + freeSpaces.setInput(new AggKeyInput("oc")); + aggMetrics.put("freeSpaces", freeSpaces); + + AggMetric occupiedSpaces = new AggMetric(); + occupiedSpaces.setFunction(AggFunction.COUNT); + occupiedSpaces.setFilter("return oc == true"); + occupiedSpaces.setInput(new AggKeyInput("oc")); + aggMetrics.put("occupiedSpaces", occupiedSpaces); + + AggMetric totalSpaces = new AggMetric(); + totalSpaces.setFunction(AggFunction.COUNT); + totalSpaces.setInput(new AggFunctionInput("return 1;")); + aggMetrics.put("totalSpaces", totalSpaces); + + Output output = new Output(); + output.setType(OutputType.TIME_SERIES); + + return createAggCf("Occupied spaces", entityId, + buildSource(EntitySearchDirection.FROM, "Contains", profiles), + Map.of("oc", new ReferencedEntityKey("occupied", ArgumentType.TS_LATEST, null)), + aggMetrics, + output); + } + + private AggSource buildSource(EntitySearchDirection direction, String relationType, List profiles) { + AggSource source = new AggSource(); + source.setRelation(new RelationPathLevel(direction, relationType)); + source.setEntityProfiles(profiles); + return source; + } + + private CalculatedField createAggCf(String name, + EntityId entityId, + AggSource aggSource, + Map inputs, + Map metrics, + Output output) { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setName(name); + calculatedField.setEntityId(entityId); + calculatedField.setType(CalculatedFieldType.LATEST_VALUES_AGGREGATION); + + LatestValuesAggregationCalculatedFieldConfiguration configuration = new LatestValuesAggregationCalculatedFieldConfiguration(); + configuration.setSource(aggSource); + configuration.setInputs(inputs); + configuration.setDeduplicationIntervalMillis(deduplicationInterval); + configuration.setMetrics(metrics); + configuration.setOutput(output); + + calculatedField.setConfiguration(configuration); + calculatedField.setDebugSettings(DebugSettings.all()); + return saveCalculatedField(calculatedField); + } + + private Device createDevice(String name, DeviceProfileId deviceProfileId, String accessToken) { + Device device = new Device(); + device.setName(name); + device.setDeviceProfileId(deviceProfileId); + DeviceData deviceData = new DeviceData(); + deviceData.setTransportConfiguration(new DefaultDeviceTransportConfiguration()); + deviceData.setConfiguration(new DefaultDeviceConfiguration()); + device.setDeviceData(deviceData); + return doPost("/api/device?accessToken=" + accessToken, device, Device.class); + } + + private Asset createAsset(String name, AssetProfileId assetProfileId) { + Asset asset = new Asset(); + asset.setName(name); + asset.setAssetProfileId(assetProfileId); + return doPost("/api/asset", asset, Asset.class); + } + + private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 2050e1075c..6c6b9c44ec 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -1063,6 +1063,16 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { doPost("/api/relation", relation); } + protected void deleteEntityRelation(EntityRelation entityRelation) throws Exception { + String url = String.format("/api/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", + entityRelation.getFrom().getId(), + entityRelation.getFrom().getEntityType(), + entityRelation.getType(), + entityRelation.getTo().getId(), + entityRelation.getTo().getEntityType()); + doDelete(url); + } + protected List findRelationsByTo(EntityId entityId) throws Exception { String url = String.format("/api/relations?toId=%s&toType=%s", entityId.getId(), entityId.getEntityType().name()); MvcResult mvcResult = doGet(url).andReturn(); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 701c22587b..6c8acbfb83 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; @@ -30,6 +31,7 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; @@ -87,6 +89,8 @@ public interface TbClusterService extends TbQueueClusterService { void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); + void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, EntityId profileId, ComponentLifecycleEvent state); + void broadcast(ComponentLifecycleMsg componentLifecycleMsg); void onDeviceProfileChange(DeviceProfile deviceProfile, DeviceProfile oldDeviceProfile, TbQueueCallback callback); @@ -137,4 +141,8 @@ public interface TbClusterService extends TbQueueClusterService { void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback); + void onRelationUpdated(TenantId tenantId, EntityRelation entityRelation, TbQueueCallback callback); + + void onRelationDeleted(TenantId tenantId, EntityRelation entityRelation, TbQueueCallback callback); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index a0bc9a72e6..5b3290c110 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -86,6 +86,14 @@ public interface RelationService { ListenableFuture> findByRelationPathQueryAsync(TenantId tenantId, EntityRelationPathQuery relationPathQuery); + List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId profileId); + + EntityRelation findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId); + + void evictRelationsByProfile(TenantId tenantId, EntityId profileId); + + void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId); + // TODO: This method may be useful for some validations in the future // ListenableFuture checkRecursiveRelation(EntityId from, EntityId to); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index 7052ab70e9..397d31650a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -24,7 +24,8 @@ public enum CalculatedFieldType { SIMPLE, SCRIPT, GEOFENCING, - ALARM; + ALARM, + LATEST_VALUES_AGGREGATION; public static final Set all = Collections.unmodifiableSet(EnumSet.allOf(CalculatedFieldType.class)); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index d3622a2dcf..4a4e3380f5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -40,7 +41,8 @@ import java.util.stream.Collectors; @Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), @Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT"), @Type(value = GeofencingCalculatedFieldConfiguration.class, name = "GEOFENCING"), - @Type(value = AlarmCalculatedFieldConfiguration.class, name = "ALARM") + @Type(value = AlarmCalculatedFieldConfiguration.class, name = "ALARM"), + @Type(value = LatestValuesAggregationCalculatedFieldConfiguration.class, name = "LATEST_VALUES_AGGREGATION") }) @JsonIgnoreProperties(ignoreUnknown = true) public interface CalculatedFieldConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunction.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunction.java new file mode 100644 index 0000000000..cd0e3f66d4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunction.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +public enum AggFunction { + MIN, MAX, SUM, AVG, COUNT, COUNT_UNIQUE +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunctionInput.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunctionInput.java new file mode 100644 index 0000000000..f6df80952e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunctionInput.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class AggFunctionInput implements AggInput { + + private String function; + + @Override + public String getType() { + return "function"; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java new file mode 100644 index 0000000000..fac988d5d9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = AggKeyInput.class, name = "key"), + @JsonSubTypes.Type(value = AggFunctionInput.class, name = "function") +}) +@JsonIgnoreProperties(ignoreUnknown = true) +public interface AggInput { + + String getType(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggKeyInput.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggKeyInput.java new file mode 100644 index 0000000000..1a00a18a9d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggKeyInput.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class AggKeyInput implements AggInput { + + private String key; + + @Override + public String getType() { + return "key"; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java new file mode 100644 index 0000000000..ebd612b1e0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Data; + +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class AggMetric { + + private AggFunction function; + private String filter; + private AggInput input; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java new file mode 100644 index 0000000000..84f6f92eb3 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.RelationPathLevel; + +import java.util.List; + +@Data +public class AggSource { + + private RelationPathLevel relation; + private List entityProfiles; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java new file mode 100644 index 0000000000..393697877e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java @@ -0,0 +1,104 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; +import java.util.function.Predicate; + +@Data +@Builder +public class CfAggTrigger { + + private List entityProfiles; + private List inputs; + + public boolean matches(EntityId profileId, Predicate cfAggTrigger) { + if (matchesProfile(profileId)) { + return cfAggTrigger.test(this); + } + return false; + } + + public boolean matchesProfile(EntityId profileId) { + return entityProfiles.contains(profileId); + } + + public boolean matchesTimeSeries(List telemetry) { + if (telemetry == null || telemetry.isEmpty()) { + return false; + } + for (TsKvEntry tsKvEntry : telemetry) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKvEntry.getKey(), ArgumentType.TS_LATEST, null); + if (inputs.contains(latestKey)) { + return true; + } + } + return false; + } + + public boolean matchesAttributes(List attributes, AttributeScope scope) { + if (attributes == null || attributes.isEmpty()) { + return false; + } + for (AttributeKvEntry attributeKvEntry : attributes) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(attributeKvEntry.getKey(), ArgumentType.ATTRIBUTE, scope); + if (inputs.contains(latestKey)) { + return true; + } + } + return false; + } + + public boolean matchesTimeSeriesKeys(List telemetry) { + if (telemetry == null || telemetry.isEmpty()) { + return false; + } + + for (String key : telemetry) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); + if (inputs.contains(latestKey)) { + return true; + } + } + + return false; + } + + public boolean matchesAttributesKeys(List attributes, AttributeScope scope) { + if (attributes == null || attributes.isEmpty()) { + return false; + } + + for (String key : attributes) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); + if (inputs.contains(latestKey)) { + return true; + } + } + + return false; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..e3db50fda0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; + +import java.util.List; +import java.util.Map; + +@Data +public class LatestValuesAggregationCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + private AggSource source; + private Map inputs; + private long deduplicationIntervalMillis; + private Map metrics; + private Output output; + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.LATEST_VALUES_AGGREGATION; + } + + @Override + public void validate() { + } + + public CfAggTrigger buildTrigger() { + return CfAggTrigger.builder() + .inputs(List.copyOf(inputs.values())) + .entityProfiles(source.getEntityProfiles()) + .build(); + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index c13b0200c7..f82de35819 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -152,6 +152,8 @@ public enum MsgType { CF_ENTITY_INIT_CF_MSG, CF_ENTITY_DELETE_MSG, + CF_RELATED_ENTITY_MSG, + CF_ARGUMENT_RESET_MSG, // Sent to reset argument; CF_REEVALUATE_MSG; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index 23b9fe08e3..38df529c96 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -47,14 +47,15 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { private final EntityId oldProfileId; private final EntityId profileId; private final boolean ownerChanged; + private final boolean relationChanged; private final JsonNode info; public ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event) { - this(tenantId, entityId, event, null, null, null, null, false, null); + this(tenantId, entityId, event, null, null, null, null, false, false, null); } @Builder - private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId, boolean ownerChanged, JsonNode info) { + private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId, boolean ownerChanged, boolean relationChanged, JsonNode info) { this.tenantId = tenantId; this.entityId = entityId; this.event = event; @@ -63,6 +64,7 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { this.oldProfileId = oldProfileId; this.profileId = profileId; this.ownerChanged = ownerChanged; + this.relationChanged = relationChanged; this.info = info; } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 26a64c7f8a..83fb158efd 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -130,6 +130,7 @@ public class ProtoUtils { builder.setOldProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); } builder.setOwnerChanged(msg.isOwnerChanged()); + builder.setRelationChanged(msg.isRelationChanged()); if (msg.getName() != null) { builder.setName(msg.getName()); } @@ -167,6 +168,7 @@ public class ProtoUtils { builder.oldProfileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB()))); } builder.ownerChanged(proto.getOwnerChanged()); + builder.relationChanged(proto.getRelationChanged()); if (proto.hasInfo()) { builder.info(JacksonUtil.toJsonNode(proto.getInfo())); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 4d99608a6d..764f4c3ec9 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -915,6 +915,11 @@ message GeofencingArgumentProto { repeated GeofencingZoneProto zones = 2; } +message AggSingleArgumentEntryProto { + EntityIdProto entityId = 1; + SingleValueArgumentProto value = 2; +} + message CalculatedFieldStateProto { CalculatedFieldEntityCtxIdProto id = 1; string type = 2; @@ -922,6 +927,7 @@ message CalculatedFieldStateProto { repeated TsRollingArgumentProto rollingValueArguments = 4; repeated GeofencingArgumentProto geofencingArguments = 5; AlarmStateProto alarmState = 6; + repeated AggSingleArgumentEntryProto aggArguments = 7; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. @@ -1292,6 +1298,7 @@ message ComponentLifecycleMsgProto { int64 profileIdLSB = 12; optional string info = 13; bool ownerChanged = 100; + bool relationChanged = 15; } message EdgeEventMsgProto { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java index 73a2183564..bf2123a26b 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java @@ -28,6 +28,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = TbelCfSingleValueArg.class, name = "SINGLE_VALUE"), @JsonSubTypes.Type(value = TbelCfTsRollingArg.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = TbelCfTsGeofencingArg.class, name = "GEOFENCING_CF_ARGUMENT_VALUE"), + @JsonSubTypes.Type(value = TbelCfLatestValuesAggregation.class, name = "LATEST_VALUES_AGGREGATION") }) public interface TbelCfArg extends TbelCfObject { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java new file mode 100644 index 0000000000..1b5fa394d2 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class TbelCfLatestValuesAggregation implements TbelCfArg { + + private final Object value; + + @JsonCreator + public TbelCfLatestValuesAggregation( + @JsonProperty("value") Object value + ) { + this.value = value; + } + + + @Override + public String getType() { + return "LATEST_VALUES_AGGREGATION"; + } + + @Override + public long memorySize() { + return 32; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 18d1806fe4..df79ca145a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -514,6 +514,40 @@ public class BaseRelationService implements RelationService { return executor.submit(() -> relationDao.findByRelationPathQuery(tenantId, relationPathQuery)); } + @Override + public List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId profileId) { + RelationCacheKey cacheKey = RelationCacheKey.builder().from(from).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.FROM).entityProfile(profileId).build(); + return cache.getAndPutInTransaction(cacheKey, + () -> relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, profileId), + RelationCacheValue::getRelations, + relations -> RelationCacheValue.builder().relations(relations).build(), false); + } + + @Override + public EntityRelation findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId) { + RelationCacheKey cacheKey = RelationCacheKey.builder().to(to).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.TO).entityProfile(profileId).build(); + return cache.getAndPutInTransaction(cacheKey, + () -> relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, profileId), + RelationCacheValue::getRelation, + relation -> RelationCacheValue.builder().relation(relation).build(), false); + } + + @Override + public void evictRelationsByProfile(TenantId tenantId, EntityId profileId) { + RelationCacheKey key = RelationCacheKey.builder().entityProfile(profileId).build(); + cache.evict(List.of(key)); + log.debug("Processed evict relations by key: {}", key); + } + + @Override + public void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId) { + List keys = new ArrayList<>(2); + keys.add(RelationCacheKey.builder().from(entityId).entityProfile(profileId).build()); + keys.add(RelationCacheKey.builder().to(entityId).entityProfile(profileId).build()); + cache.evict(keys); + log.debug("Processed evict relations by keys: {}", keys); + } + private void validate(EntityRelationPathQuery relationPathQuery) { validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); List levels = relationPathQuery.levels(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java index d6f0525c9d..344af0a6f3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java @@ -40,9 +40,14 @@ public class RelationCacheKey implements Serializable { private final String type; private final RelationTypeGroup typeGroup; private final EntitySearchDirection direction; + private final EntityId entityProfile; public RelationCacheKey(EntityId from, EntityId to, String type, RelationTypeGroup typeGroup) { - this(from, to, type, typeGroup, null); + this(from, to, type, typeGroup, null, null); + } + + public RelationCacheKey(EntityId from, EntityId to, String type, RelationTypeGroup typeGroup, EntitySearchDirection direction) { + this(from, to, type, typeGroup, direction, null); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index ad53164ad7..f529396965 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -36,8 +36,12 @@ public interface RelationDao { List findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup); + List findByFromAndTypeAndProfile(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup, EntityId profileId); + List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup); + EntityRelation findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId); + List findAllByTo(TenantId tenantId, EntityId to); List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index b2871313ed..c859b71580 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -103,6 +103,11 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override + public List findByFromAndTypeAndProfile(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { + return DaoUtil.convertDataList(relationRepository.findByFromAndProfile(from.getId(), from.getEntityType().name(), typeGroup.name(), relationType, profileId.getId())); + } + @Override public List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup) { return DaoUtil.convertDataList( @@ -112,6 +117,11 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override + public EntityRelation findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { + return DaoUtil.getData(relationRepository.findByToAndProfile(to.getId(), to.getEntityType().name(), typeGroup.name(), relationType, profileId.getId())); + } + @Override public List findAllByTo(TenantId tenantId, EntityId to) { return DaoUtil.convertDataList( diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index b4e8a21372..9294236526 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.sql.relation; -import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; @@ -28,6 +27,7 @@ import org.thingsboard.server.dao.model.sql.RelationCompositeKey; import org.thingsboard.server.dao.model.sql.RelationEntity; import java.util.List; +import java.util.Optional; import java.util.UUID; public interface RelationRepository @@ -96,4 +96,41 @@ public interface RelationRepository @Param("toId") UUID toId, @Param("toType") String toType, @Param("batchSize") int batchSize); + + @Query(value = """ + SELECT r.from_id, r.from_type, r.relation_type_group, r.relation_type, r.to_id, r.to_type, r.additional_info, r.version + FROM relation r + LEFT JOIN device d ON r.to_id = d.id AND r.to_type = 'DEVICE' + LEFT JOIN asset a ON r.to_id = a.id AND r.to_type = 'ASSET' + WHERE r.from_id = :fromId + AND r.from_type = :fromType + AND r.relation_type = :relationType + AND r.relation_type_group = :relationTypeGroup + AND ((d.device_profile_id = :profileId) OR (a.asset_profile_id = :profileId)) + AND (d.id IS NOT NULL OR a.id IS NOT NULL) + """, nativeQuery = true) + List findByFromAndProfile(@Param("fromId") UUID fromId, + @Param("fromType") String fromType, + @Param("relationTypeGroup") String relationTypeGroup, + @Param("relationType") String relationType, + @Param("profileId") UUID profileId); + + @Query(value = """ + SELECT r.from_id, r.from_type, r.relation_type_group, r.relation_type, r.to_id, r.to_type, r.additional_info, r.version + FROM relation r + LEFT JOIN device d ON r.from_id = d.id AND r.from_type = 'DEVICE' + LEFT JOIN asset a ON r.from_id = a.id AND r.from_type = 'ASSET' + WHERE r.to_id = :toId + AND r.to_type = :toType + AND r.relation_type = :relationType + AND r.relation_type_group = :relationTypeGroup + AND ((d.device_profile_id = :profileId) OR (a.asset_profile_id = :profileId)) + AND (d.id IS NOT NULL OR a.id IS NOT NULL) + LIMIT 1 + """, nativeQuery = true) + Optional findByToAndProfile(@Param("toId") UUID toId, + @Param("toType") String toType, + @Param("relationTypeGroup") String relationTypeGroup, + @Param("relationType") String relationType, + @Param("profileId") UUID profileId); }