From 9f208b4abd7628de30f633493bece143fb6b7684 Mon Sep 17 00:00:00 2001 From: dshvaika Date: Thu, 9 Oct 2025 13:02:02 +0300 Subject: [PATCH] Added related entities limit + additional validation for CF configuration --- .../main/data/upgrade/basic/schema_update.sql | 10 +- .../cf/ctx/state/CalculatedFieldCtx.java | 8 +- ...opagationCalculatedFieldConfiguration.java | 3 + .../DefaultTenantProfileConfiguration.java | 4 +- ...ationCalculatedFieldConfigurationTest.java | 24 ++++ .../dao/relation/BaseRelationService.java | 15 ++- .../server/dao/relation/RelationDao.java | 2 +- .../dao/sql/relation/JpaRelationDao.java | 20 ++- .../dao/service/RelationServiceTest.java | 117 ++++++++++++++---- 9 files changed, 160 insertions(+), 43 deletions(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 0add4c0545..50c44c0e2d 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -34,7 +34,13 @@ SET profile_data = jsonb_set( WHEN (profile_data -> 'configuration') ? 'maxRelationLevelPerCfArgument' THEN NULL ELSE to_jsonb(10) - END + END, + 'maxRelatedEntitiesToReturnPerCfArgument', + CASE + WHEN (profile_data -> 'configuration') ? 'maxRelatedEntitiesToReturnPerCfArgument' + THEN NULL + ELSE to_jsonb(100) + END, ) ), false @@ -43,6 +49,8 @@ WHERE NOT ( (profile_data -> 'configuration') ? 'minAllowedScheduledUpdateIntervalInSecForCF' AND (profile_data -> 'configuration') ? 'maxRelationLevelPerCfArgument' + AND + (profile_data -> 'configuration') ? 'maxRelatedEntitiesToReturnPerCfArgument' ); -- UPDATE TENANT PROFILE CONFIGURATION END diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index bfb6483d18..5b6b708d26 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -498,12 +498,8 @@ public class CalculatedFieldCtx { } public boolean hasContextOnlyChanges(CalculatedFieldCtx other) { // has changes that do not require state reinit and will be picked up by the state on the fly - if (calculatedField.getConfiguration() instanceof ExpressionBasedCalculatedFieldConfiguration expressionConfig) { - boolean shouldCompareExpression = !(expressionConfig instanceof PropagationCalculatedFieldConfiguration propagationConfig) - || propagationConfig.isApplyExpressionToResolvedArguments(); - if (shouldCompareExpression && !expression.equals(other.expression)) { - return true; - } + if (calculatedField.getConfiguration() instanceof ExpressionBasedCalculatedFieldConfiguration && !Objects.equals(expression, other.expression)) { + return true; } if (!output.equals(other.output)) { return true; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java index 3394813d3f..70ffd952cf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java @@ -50,6 +50,9 @@ public class PropagationCalculatedFieldConfiguration extends BaseCalculatedField propagationRestriction(); if (!applyExpressionToResolvedArguments) { arguments.forEach((name, argument) -> { + if (argument.getRefEntityId() != null || argument.getRefDynamicSourceConfiguration() != null) { + throw new IllegalArgumentException("Arguments in 'Arguments only' propagation mode support only the 'Current entity' source entity type!"); + } if (argument.getRefEntityKey() == null) { throw new IllegalArgumentException("Argument: '" + name + "' doesn't have reference entity key configured!"); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index c6bd9a7f38..5d1c4f7018 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -172,10 +172,12 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private long maxCalculatedFieldsPerEntity = 5; @Schema(example = "10") private long maxArgumentsPerCF = 10; - @Schema(example = "3600") + @Schema(example = "60") private int minAllowedScheduledUpdateIntervalInSecForCF = 60; @Schema(example = "10") private int maxRelationLevelPerCfArgument = 10; + @Schema(example = "100") + private int maxRelatedEntitiesToReturnPerCfArgument = 100; @Builder.Default @Min(value = 1, message = "must be at least 1") @Schema(example = "1000") diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java index eb0591e32b..64f7b0efd0 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java @@ -19,10 +19,12 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import java.util.Map; +import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -37,6 +39,28 @@ public class PropagationCalculatedFieldConfigurationTest { assertThat(cfg.getType()).isEqualTo(CalculatedFieldType.PROPAGATION); } + @Test + void validateShouldThrowWhenConfigurationDisallowArgumentsWithReferencedEntity() { + var cfg = new PropagationCalculatedFieldConfiguration(); + Argument argumentWithRefEntityIdSet = new Argument(); + argumentWithRefEntityIdSet.setRefEntityId(new DeviceId(UUID.fromString("bda14084-f40e-4acc-9b85-9d1dd209bb64"))); + cfg.setArguments(Map.of("argumentWithRefEntityIdSet", argumentWithRefEntityIdSet)); + assertThatThrownBy(cfg::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Arguments in 'Arguments only' propagation mode support only the 'Current entity' source entity type!"); + } + + @Test + void validateShouldThrowWhenConfigurationDisallowArgumentsWithDynamicReferenceConfiguration() { + var cfg = new PropagationCalculatedFieldConfiguration(); + Argument argumentWithRefEntityIdSet = new Argument(); + argumentWithRefEntityIdSet.setRefDynamicSourceConfiguration(new CurrentOwnerDynamicSourceConfiguration()); + cfg.setArguments(Map.of("argumentWithRefEntityIdSet", argumentWithRefEntityIdSet)); + assertThatThrownBy(cfg::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Arguments in 'Arguments only' propagation mode support only the 'Current entity' source entity type!"); + } + @Test void validateShouldThrowWhenUsedReservedPropagationArgumentName() { var cfg = new PropagationCalculatedFieldConfiguration(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 18d1806fe4..34928c7e04 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -50,12 +50,14 @@ import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.relation.RelationsSearchParameters; import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.ConstraintValidator; import org.thingsboard.server.dao.sql.JpaExecutorService; import org.thingsboard.server.dao.sql.relation.JpaRelationQueryExecutorService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import java.util.ArrayList; import java.util.Collections; @@ -71,6 +73,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import static org.thingsboard.server.dao.service.Validator.validateId; +import static org.thingsboard.server.dao.service.Validator.validatePositiveNumber; /** * Created by ashvayka on 28.04.17. @@ -85,6 +88,8 @@ public class BaseRelationService implements RelationService { private final ApplicationEventPublisher eventPublisher; private final JpaExecutorService executor; private final JpaRelationQueryExecutorService relationsExecutor; + private final ApiLimitService apiLimitService; + protected ScheduledExecutorService timeoutExecutorService; @Value("${sql.relations.query_timeout:20}") @@ -93,13 +98,14 @@ public class BaseRelationService implements RelationService { public BaseRelationService(RelationDao relationDao, @Lazy EntityService entityService, TbTransactionalCache cache, ApplicationEventPublisher eventPublisher, JpaExecutorService executor, - JpaRelationQueryExecutorService relationsExecutor) { + JpaRelationQueryExecutorService relationsExecutor, ApiLimitService apiLimitService) { this.relationDao = relationDao; this.entityService = entityService; this.cache = cache; this.eventPublisher = eventPublisher; this.executor = executor; this.relationsExecutor = relationsExecutor; + this.apiLimitService = apiLimitService; } @PostConstruct @@ -504,14 +510,17 @@ public class BaseRelationService implements RelationService { log.trace("Executing findByRelationPathQuery, tenantId [{}], relationPathQuery {}", tenantId, relationPathQuery); validateId(tenantId, id -> "Invalid tenant id: " + id); validate(relationPathQuery); + int limit = (int) apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxRelatedEntitiesToReturnPerCfArgument); + validatePositiveNumber(limit, "Invalid entities limit: " + limit); if (relationPathQuery.levels().size() == 1) { RelationPathLevel relationPathLevel = relationPathQuery.levels().get(0); - return switch (relationPathLevel.direction()) { + var relationsFuture = switch (relationPathLevel.direction()) { case FROM -> findByFromAndTypeAsync(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), RelationTypeGroup.COMMON); case TO -> findByToAndTypeAsync(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), RelationTypeGroup.COMMON); }; + return Futures.transform(relationsFuture, entityRelations -> entityRelations.subList(0, limit), MoreExecutors.directExecutor()); } - return executor.submit(() -> relationDao.findByRelationPathQuery(tenantId, relationPathQuery)); + return executor.submit(() -> relationDao.findByRelationPathQuery(tenantId, relationPathQuery, limit)); } private void validate(EntityRelationPathQuery relationPathQuery) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index ad53164ad7..2ec23a0d74 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -72,6 +72,6 @@ public interface RelationDao { List findRuleNodeToRuleChainRelations(RuleChainType ruleChainType, int limit); - List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery relationPathQuery); + List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery relationPathQuery, int limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index b2871313ed..4c1b8f299b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -299,15 +299,18 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple } @Override - public List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery query) { + public List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery query, int limit) { List levels = query.levels(); if (levels == null || levels.isEmpty()) { - return Collections.emptyList(); + return List.of(); + } + if (limit <= 0) { + return List.of(); } String sql = buildRelationPathSql(query); - Object[] params = buildRelationPathParams(query); + Object[] params = buildRelationPathParams(query, limit); - log.trace("[{}] relation path query: {}", tenantId, sql); + log.info("[{}] relation path query: {}", tenantId, sql); return jdbcTemplate.queryForList(sql, params).stream() .map(row -> { @@ -330,7 +333,7 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple .collect(Collectors.toList()); } - private Object[] buildRelationPathParams(EntityRelationPathQuery query) { + private Object[] buildRelationPathParams(EntityRelationPathQuery query, int limit) { final List params = new ArrayList<>(); // seed params.add(query.rootEntityId().getId()); @@ -340,6 +343,10 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple for (var lvl : query.levels()) { params.add(lvl.relationType()); } + + // limit + params.add(limit); + return params.toArray(); } @@ -387,7 +394,8 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple .append("FROM ").append(RELATION_TABLE_NAME).append(" r\n") .append("JOIN ").append(prevForLast).append(" p ON ").append(lastJoin).append("\n") .append("WHERE r.relation_type_group = '").append(RelationTypeGroup.COMMON).append("'\n") - .append(" AND r.relation_type = ?"); + .append(" AND r.relation_type = ?\n") + .append("LIMIT ?"); return sb.toString(); } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/RelationServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/RelationServiceTest.java index bb7bfad677..e3827d6a6e 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/RelationServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/RelationServiceTest.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.relation.RelationsSearchParameters; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import java.util.ArrayList; import java.util.Collections; @@ -52,6 +53,9 @@ public class RelationServiceTest extends AbstractServiceTest { @Autowired RelationService relationService; + @Autowired + private TbTenantProfileCache tbTenantProfileCache; + @Before public void before() { } @@ -628,48 +632,111 @@ public class RelationServiceTest extends AbstractServiceTest { } @Test - public void testFindByPathQuery() throws Exception { + public void testFindByPathQueryWithoutExceedingLimit() throws Exception { /* A └──[firstLevel, TO]→ B └──[secondLevel, TO]→ C - ├──[thirdLevel, FROM]→ D - ├──[thirdLevel, FROM]→ E - └──[thirdLevel, FROM]→ F + ├──[thirdLevel, FROM]→ D1 + ├──[thirdLevel, FROM]→ D2 + ├──[thirdLevel, FROM]→ ... + └──[thirdLevel, FROM]→ D{N - 1}, where N is the limit */ - // rootEntity AssetId assetA = new AssetId(Uuids.timeBased()); - // firstLevelEntity AssetId assetB = new AssetId(Uuids.timeBased()); - // secondLevelEntity AssetId assetC = new AssetId(Uuids.timeBased()); - // thirdLevelEntities - AssetId assetD = new AssetId(Uuids.timeBased()); - AssetId assetE = new AssetId(Uuids.timeBased()); - AssetId assetF = new AssetId(Uuids.timeBased()); - EntityRelation firstLevelRelation = new EntityRelation(assetB, assetA, "firstLevel"); - EntityRelation secondLevelRelation = new EntityRelation(assetC, assetB, "secondLevel"); - EntityRelation thirdLevelRelation1 = new EntityRelation(assetC, assetD, "thirdLevel"); - EntityRelation thirdLevelRelation2 = new EntityRelation(assetC, assetE, "thirdLevel"); - EntityRelation thirdLevelRelation3 = new EntityRelation(assetC, assetF, "thirdLevel"); + // create first and second level + saveRelation(new EntityRelation(assetB, assetA, "firstLevel")); + saveRelation(new EntityRelation(assetC, assetB, "secondLevel")); - firstLevelRelation = saveRelation(firstLevelRelation); - secondLevelRelation = saveRelation(secondLevelRelation); - thirdLevelRelation1 = saveRelation(thirdLevelRelation1); - thirdLevelRelation2 = saveRelation(thirdLevelRelation2); - thirdLevelRelation3 = saveRelation(thirdLevelRelation3); + int limit = tbTenantProfileCache.get(tenantId) + .getDefaultProfileConfiguration() + .getMaxRelatedEntitiesToReturnPerCfArgument(); - List expectedRelations = List.of(thirdLevelRelation1, thirdLevelRelation2, thirdLevelRelation3); + int totalCreated = limit - 1; - EntityRelationPathQuery relationPathQuery = new EntityRelationPathQuery(assetA, List.of( + List allThirdLevelRelations = new ArrayList<>(); + for (int i = 0; i < totalCreated; i++) { + AssetId leaf = new AssetId(Uuids.timeBased()); + allThirdLevelRelations.add(saveRelation(new EntityRelation(assetC, leaf, "thirdLevel"))); + } + + EntityRelationPathQuery query = new EntityRelationPathQuery(assetA, List.of( new RelationPathLevel(EntitySearchDirection.TO, "firstLevel"), new RelationPathLevel(EntitySearchDirection.TO, "secondLevel"), new RelationPathLevel(EntitySearchDirection.FROM, "thirdLevel") )); - List entityRelations = relationService.findByRelationPathQueryAsync(tenantId, relationPathQuery).get(); - assertThat(expectedRelations).containsExactlyInAnyOrderElementsOf(entityRelations); + // call a method that applies the default limit internally + List result = relationService.findByRelationPathQueryAsync(tenantId, query).get(); + + // verify that limit has been applied + assertThat(result).hasSize(totalCreated); + + // verify all returned are valid third-level relations under C + assertThat(result) + .allSatisfy(rel -> { + assertThat(rel.getType()).isEqualTo("thirdLevel"); + assertThat(rel.getFrom()).isEqualTo(assetC); + }); + + // verify the returned subset is part of all created relations + assertThat(result).isEqualTo(allThirdLevelRelations); + } + + @Test + public void testFindByPathQueryWithExceedingLimit() throws Exception { + /* + A + └──[firstLevel, TO]→ B + └──[secondLevel, TO]→ C + ├──[thirdLevel, FROM]→ D1 + ├──[thirdLevel, FROM]→ D2 + ├──[thirdLevel, FROM]→ ... + └──[thirdLevel, FROM]→ D{N + 20}, where N is the limit + */ + AssetId assetA = new AssetId(Uuids.timeBased()); + AssetId assetB = new AssetId(Uuids.timeBased()); + AssetId assetC = new AssetId(Uuids.timeBased()); + + // create first and second level + saveRelation(new EntityRelation(assetB, assetA, "firstLevel")); + saveRelation(new EntityRelation(assetC, assetB, "secondLevel")); + + int limit = tbTenantProfileCache.get(tenantId) + .getDefaultProfileConfiguration() + .getMaxRelatedEntitiesToReturnPerCfArgument(); + + int totalCreated = limit + 20; + + List allThirdLevelRelations = new ArrayList<>(); + for (int i = 0; i < totalCreated; i++) { + AssetId leaf = new AssetId(Uuids.timeBased()); + allThirdLevelRelations.add(saveRelation(new EntityRelation(assetC, leaf, "thirdLevel"))); + } + + EntityRelationPathQuery query = new EntityRelationPathQuery(assetA, List.of( + new RelationPathLevel(EntitySearchDirection.TO, "firstLevel"), + new RelationPathLevel(EntitySearchDirection.TO, "secondLevel"), + new RelationPathLevel(EntitySearchDirection.FROM, "thirdLevel") + )); + + // call a method that applies the default limit internally + List result = relationService.findByRelationPathQueryAsync(tenantId, query).get(); + + // verify that limit has been applied + assertThat(result).hasSize(limit); + + // verify all returned are valid third-level relations under C + assertThat(result) + .allSatisfy(rel -> { + assertThat(rel.getType()).isEqualTo("thirdLevel"); + assertThat(rel.getFrom()).isEqualTo(assetC); + }); + + // verify the returned subset is part of all created relations + assertThat(result).isSubsetOf(allThirdLevelRelations); } @Test