Added related entities limit + additional validation for CF configuration

application/src/main/data/upgrade/basic/schema_update.sql

@@ -34,7 +34,13 @@ SET profile_data = jsonb_set(
                             WHEN (profile_data -> 'configuration') ? 'maxRelationLevelPerCfArgument'
                                 THEN NULL
                             ELSE to_jsonb(10)
-                            END
+                            END,
+                        'maxRelatedEntitiesToReturnPerCfArgument',
+                        CASE
+                            WHEN (profile_data -> 'configuration') ? 'maxRelatedEntitiesToReturnPerCfArgument'
+                                THEN NULL
+                            ELSE to_jsonb(100)
+                            END,
                 )
                ),
         false
@@ -43,6 +49,8 @@ WHERE NOT (
     (profile_data -> 'configuration') ? 'minAllowedScheduledUpdateIntervalInSecForCF'
         AND
     (profile_data -> 'configuration') ? 'maxRelationLevelPerCfArgument'
+        AND
+    (profile_data -> 'configuration') ? 'maxRelatedEntitiesToReturnPerCfArgument'
     );
 
 -- UPDATE TENANT PROFILE CONFIGURATION END

application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java

@@ -498,12 +498,8 @@ public class CalculatedFieldCtx {
     }
 
     public boolean hasContextOnlyChanges(CalculatedFieldCtx other) { // has changes that do not require state reinit and will be picked up by the state on the fly
-        if (calculatedField.getConfiguration() instanceof ExpressionBasedCalculatedFieldConfiguration expressionConfig) {
+        if (calculatedField.getConfiguration() instanceof ExpressionBasedCalculatedFieldConfiguration && !Objects.equals(expression, other.expression)) {
-            boolean shouldCompareExpression = !(expressionConfig instanceof PropagationCalculatedFieldConfiguration propagationConfig)
+            return true;
-                                              || propagationConfig.isApplyExpressionToResolvedArguments();
-            if (shouldCompareExpression && !expression.equals(other.expression)) {
-                return true;
-            }
         }
         if (!output.equals(other.output)) {
             return true;

common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfiguration.java

@@ -50,6 +50,9 @@ public class PropagationCalculatedFieldConfiguration extends BaseCalculatedField
         propagationRestriction();
         if (!applyExpressionToResolvedArguments) {
             arguments.forEach((name, argument) -> {
+                if (argument.getRefEntityId() != null || argument.getRefDynamicSourceConfiguration() != null) {
+                    throw new IllegalArgumentException("Arguments in 'Arguments only' propagation mode support only the 'Current entity' source entity type!");
+                }
                 if (argument.getRefEntityKey() == null) {
                     throw new IllegalArgumentException("Argument: '" + name + "' doesn't have reference entity key configured!");
                 }

common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java

@@ -172,10 +172,12 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura
     private long maxCalculatedFieldsPerEntity = 5;
     @Schema(example = "10")
     private long maxArgumentsPerCF = 10;
-    @Schema(example = "3600")
+    @Schema(example = "60")
     private int minAllowedScheduledUpdateIntervalInSecForCF = 60;
     @Schema(example = "10")
     private int maxRelationLevelPerCfArgument = 10;
+    @Schema(example = "100")
+    private int maxRelatedEntitiesToReturnPerCfArgument = 100;
     @Builder.Default
     @Min(value = 1, message = "must be at least 1")
     @Schema(example = "1000")

common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/PropagationCalculatedFieldConfigurationTest.java

@@ -19,10 +19,12 @@ import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.thingsboard.server.common.data.cf.CalculatedFieldType;
+import org.thingsboard.server.common.data.id.DeviceId;
 import org.thingsboard.server.common.data.relation.EntityRelation;
 import org.thingsboard.server.common.data.relation.EntitySearchDirection;
 
 import java.util.Map;
+import java.util.UUID;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -37,6 +39,28 @@ public class PropagationCalculatedFieldConfigurationTest {
         assertThat(cfg.getType()).isEqualTo(CalculatedFieldType.PROPAGATION);
     }
 
+    @Test
+    void validateShouldThrowWhenConfigurationDisallowArgumentsWithReferencedEntity() {
+        var cfg = new PropagationCalculatedFieldConfiguration();
+        Argument argumentWithRefEntityIdSet = new Argument();
+        argumentWithRefEntityIdSet.setRefEntityId(new DeviceId(UUID.fromString("bda14084-f40e-4acc-9b85-9d1dd209bb64")));
+        cfg.setArguments(Map.of("argumentWithRefEntityIdSet", argumentWithRefEntityIdSet));
+        assertThatThrownBy(cfg::validate)
+                .isInstanceOf(IllegalArgumentException.class)
+                .hasMessage("Arguments in 'Arguments only' propagation mode support only the 'Current entity' source entity type!");
+    }
+
+    @Test
+    void validateShouldThrowWhenConfigurationDisallowArgumentsWithDynamicReferenceConfiguration() {
+        var cfg = new PropagationCalculatedFieldConfiguration();
+        Argument argumentWithRefEntityIdSet = new Argument();
+        argumentWithRefEntityIdSet.setRefDynamicSourceConfiguration(new CurrentOwnerDynamicSourceConfiguration());
+        cfg.setArguments(Map.of("argumentWithRefEntityIdSet", argumentWithRefEntityIdSet));
+        assertThatThrownBy(cfg::validate)
+                .isInstanceOf(IllegalArgumentException.class)
+                .hasMessage("Arguments in 'Arguments only' propagation mode support only the 'Current entity' source entity type!");
+    }
+
     @Test
     void validateShouldThrowWhenUsedReservedPropagationArgumentName() {
         var cfg = new PropagationCalculatedFieldConfiguration();

dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java

@@ -50,12 +50,14 @@ import org.thingsboard.server.common.data.relation.RelationPathLevel;
 import org.thingsboard.server.common.data.relation.RelationTypeGroup;
 import org.thingsboard.server.common.data.relation.RelationsSearchParameters;
 import org.thingsboard.server.common.data.rule.RuleChainType;
+import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
 import org.thingsboard.server.dao.entity.EntityService;
 import org.thingsboard.server.dao.eventsourcing.RelationActionEvent;
 import org.thingsboard.server.dao.exception.DataValidationException;
 import org.thingsboard.server.dao.service.ConstraintValidator;
 import org.thingsboard.server.dao.sql.JpaExecutorService;
 import org.thingsboard.server.dao.sql.relation.JpaRelationQueryExecutorService;
+import org.thingsboard.server.dao.usagerecord.ApiLimitService;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -71,6 +73,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
 
 import static org.thingsboard.server.dao.service.Validator.validateId;
+import static org.thingsboard.server.dao.service.Validator.validatePositiveNumber;
 
 /**
  * Created by ashvayka on 28.04.17.
@@ -85,6 +88,8 @@ public class BaseRelationService implements RelationService {
     private final ApplicationEventPublisher eventPublisher;
     private final JpaExecutorService executor;
     private final JpaRelationQueryExecutorService relationsExecutor;
+    private final ApiLimitService apiLimitService;
+
     protected ScheduledExecutorService timeoutExecutorService;
 
     @Value("${sql.relations.query_timeout:20}")
@@ -93,13 +98,14 @@ public class BaseRelationService implements RelationService {
     public BaseRelationService(RelationDao relationDao, @Lazy EntityService entityService,
                                TbTransactionalCache<RelationCacheKey, RelationCacheValue> cache,
                                ApplicationEventPublisher eventPublisher, JpaExecutorService executor,
-                               JpaRelationQueryExecutorService relationsExecutor) {
+                               JpaRelationQueryExecutorService relationsExecutor, ApiLimitService apiLimitService) {
         this.relationDao = relationDao;
         this.entityService = entityService;
         this.cache = cache;
         this.eventPublisher = eventPublisher;
         this.executor = executor;
         this.relationsExecutor = relationsExecutor;
+        this.apiLimitService = apiLimitService;
     }
 
     @PostConstruct
@@ -504,14 +510,17 @@ public class BaseRelationService implements RelationService {
         log.trace("Executing findByRelationPathQuery, tenantId [{}], relationPathQuery {}", tenantId, relationPathQuery);
         validateId(tenantId, id -> "Invalid tenant id: " + id);
         validate(relationPathQuery);
+        int limit = (int) apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxRelatedEntitiesToReturnPerCfArgument);
+        validatePositiveNumber(limit, "Invalid entities limit: " + limit);
         if (relationPathQuery.levels().size() == 1) {
             RelationPathLevel relationPathLevel = relationPathQuery.levels().get(0);
-            return switch (relationPathLevel.direction()) {
+            var relationsFuture = switch (relationPathLevel.direction()) {
                 case FROM -> findByFromAndTypeAsync(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), RelationTypeGroup.COMMON);
                 case TO -> findByToAndTypeAsync(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), RelationTypeGroup.COMMON);
             };
+            return Futures.transform(relationsFuture, entityRelations -> entityRelations.subList(0, limit), MoreExecutors.directExecutor());
         }
-        return executor.submit(() -> relationDao.findByRelationPathQuery(tenantId, relationPathQuery));
+        return executor.submit(() -> relationDao.findByRelationPathQuery(tenantId, relationPathQuery, limit));
     }
 
     private void validate(EntityRelationPathQuery relationPathQuery) {

dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java

@@ -72,6 +72,6 @@ public interface RelationDao {
 
     List<EntityRelation> findRuleNodeToRuleChainRelations(RuleChainType ruleChainType, int limit);
 
-    List<EntityRelation> findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery relationPathQuery);
+    List<EntityRelation> findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery relationPathQuery, int limit);
 
 }

dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java

@@ -299,15 +299,18 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple
     }
 
     @Override
-    public List<EntityRelation> findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery query) {
+    public List<EntityRelation> findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery query, int limit) {
         List<RelationPathLevel> levels = query.levels();
         if (levels == null || levels.isEmpty()) {
-            return Collections.emptyList();
+            return List.of();
+        }
+        if (limit <= 0) {
+            return List.of();
         }
         String sql = buildRelationPathSql(query);
-        Object[] params = buildRelationPathParams(query);
+        Object[] params = buildRelationPathParams(query, limit);
 
-        log.trace("[{}] relation path query: {}", tenantId, sql);
+        log.info("[{}] relation path query: {}", tenantId, sql);
 
         return jdbcTemplate.queryForList(sql, params).stream()
                 .map(row -> {
@@ -330,7 +333,7 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple
                 .collect(Collectors.toList());
     }
 
-    private Object[] buildRelationPathParams(EntityRelationPathQuery query) {
+    private Object[] buildRelationPathParams(EntityRelationPathQuery query, int limit) {
         final List<Object> params = new ArrayList<>();
         // seed
         params.add(query.rootEntityId().getId());
@@ -340,6 +343,10 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple
         for (var lvl : query.levels()) {
             params.add(lvl.relationType());
         }
+
+        // limit
+        params.add(limit);
+
         return params.toArray();
     }
 
@@ -387,7 +394,8 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple
                 .append("FROM ").append(RELATION_TABLE_NAME).append(" r\n")
                 .append("JOIN ").append(prevForLast).append(" p ON ").append(lastJoin).append("\n")
                 .append("WHERE r.relation_type_group = '").append(RelationTypeGroup.COMMON).append("'\n")
-                .append("  AND r.relation_type = ?");
+                .append("  AND r.relation_type = ?\n")
+                .append("LIMIT ?");
 
         return sb.toString();
     }

dao/src/test/java/org/thingsboard/server/dao/service/RelationServiceTest.java

@@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup;
 import org.thingsboard.server.common.data.relation.RelationsSearchParameters;
 import org.thingsboard.server.dao.exception.DataValidationException;
 import org.thingsboard.server.dao.relation.RelationService;
+import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -52,6 +53,9 @@ public class RelationServiceTest extends AbstractServiceTest {
     @Autowired
     RelationService relationService;
 
+    @Autowired
+    private TbTenantProfileCache tbTenantProfileCache;
+
     @Before
     public void before() {
     }
@@ -628,48 +632,111 @@ public class RelationServiceTest extends AbstractServiceTest {
     }
 
     @Test
-    public void testFindByPathQuery() throws Exception {
+    public void testFindByPathQueryWithoutExceedingLimit() throws Exception {
         /*
         A
         └──[firstLevel, TO]→ B
             └──[secondLevel, TO]→ C
-                ├──[thirdLevel, FROM]→ D
+                ├──[thirdLevel, FROM]→ D1
-                ├──[thirdLevel, FROM]→ E
+                ├──[thirdLevel, FROM]→ D2
-                └──[thirdLevel, FROM]→ F
+                ├──[thirdLevel, FROM]→ ...
+                └──[thirdLevel, FROM]→ D{N - 1}, where N is the limit
         */
-        // rootEntity
         AssetId assetA = new AssetId(Uuids.timeBased());
-        // firstLevelEntity
         AssetId assetB = new AssetId(Uuids.timeBased());
-        // secondLevelEntity
         AssetId assetC = new AssetId(Uuids.timeBased());
-        // thirdLevelEntities
-        AssetId assetD = new AssetId(Uuids.timeBased());
-        AssetId assetE = new AssetId(Uuids.timeBased());
-        AssetId assetF = new AssetId(Uuids.timeBased());
 
-        EntityRelation firstLevelRelation = new EntityRelation(assetB, assetA, "firstLevel");
+        // create first and second level
-        EntityRelation secondLevelRelation = new EntityRelation(assetC, assetB, "secondLevel");
+        saveRelation(new EntityRelation(assetB, assetA, "firstLevel"));
-        EntityRelation thirdLevelRelation1 = new EntityRelation(assetC, assetD, "thirdLevel");
+        saveRelation(new EntityRelation(assetC, assetB, "secondLevel"));
-        EntityRelation thirdLevelRelation2 = new EntityRelation(assetC, assetE, "thirdLevel");
-        EntityRelation thirdLevelRelation3 = new EntityRelation(assetC, assetF, "thirdLevel");
 
-        firstLevelRelation = saveRelation(firstLevelRelation);
+        int limit = tbTenantProfileCache.get(tenantId)
-        secondLevelRelation = saveRelation(secondLevelRelation);
+                .getDefaultProfileConfiguration()
-        thirdLevelRelation1 = saveRelation(thirdLevelRelation1);
+                .getMaxRelatedEntitiesToReturnPerCfArgument();
-        thirdLevelRelation2 = saveRelation(thirdLevelRelation2);
-        thirdLevelRelation3 = saveRelation(thirdLevelRelation3);
 
-        List<EntityRelation> expectedRelations = List.of(thirdLevelRelation1, thirdLevelRelation2, thirdLevelRelation3);
+        int totalCreated = limit - 1;
 
-        EntityRelationPathQuery relationPathQuery = new EntityRelationPathQuery(assetA, List.of(
+        List<EntityRelation> allThirdLevelRelations = new ArrayList<>();
+        for (int i = 0; i < totalCreated; i++) {
+            AssetId leaf = new AssetId(Uuids.timeBased());
+            allThirdLevelRelations.add(saveRelation(new EntityRelation(assetC, leaf, "thirdLevel")));
+        }
+
+        EntityRelationPathQuery query = new EntityRelationPathQuery(assetA, List.of(
                 new RelationPathLevel(EntitySearchDirection.TO, "firstLevel"),
                 new RelationPathLevel(EntitySearchDirection.TO, "secondLevel"),
                 new RelationPathLevel(EntitySearchDirection.FROM, "thirdLevel")
         ));
-        List<EntityRelation> entityRelations = relationService.findByRelationPathQueryAsync(tenantId, relationPathQuery).get();
 
-        assertThat(expectedRelations).containsExactlyInAnyOrderElementsOf(entityRelations);
+        // call a method that applies the default limit internally
+        List<EntityRelation> result = relationService.findByRelationPathQueryAsync(tenantId, query).get();
+
+        // verify that limit has been applied
+        assertThat(result).hasSize(totalCreated);
+
+        // verify all returned are valid third-level relations under C
+        assertThat(result)
+                .allSatisfy(rel -> {
+                    assertThat(rel.getType()).isEqualTo("thirdLevel");
+                    assertThat(rel.getFrom()).isEqualTo(assetC);
+                });
+
+        // verify the returned subset is part of all created relations
+        assertThat(result).isEqualTo(allThirdLevelRelations);
+    }
+
+    @Test
+    public void testFindByPathQueryWithExceedingLimit() throws Exception {
+        /*
+        A
+        └──[firstLevel, TO]→ B
+            └──[secondLevel, TO]→ C
+                ├──[thirdLevel, FROM]→ D1
+                ├──[thirdLevel, FROM]→ D2
+                ├──[thirdLevel, FROM]→ ...
+                └──[thirdLevel, FROM]→ D{N + 20}, where N is the limit
+        */
+        AssetId assetA = new AssetId(Uuids.timeBased());
+        AssetId assetB = new AssetId(Uuids.timeBased());
+        AssetId assetC = new AssetId(Uuids.timeBased());
+
+        // create first and second level
+        saveRelation(new EntityRelation(assetB, assetA, "firstLevel"));
+        saveRelation(new EntityRelation(assetC, assetB, "secondLevel"));
+
+        int limit = tbTenantProfileCache.get(tenantId)
+                .getDefaultProfileConfiguration()
+                .getMaxRelatedEntitiesToReturnPerCfArgument();
+
+        int totalCreated = limit + 20;
+
+        List<EntityRelation> allThirdLevelRelations = new ArrayList<>();
+        for (int i = 0; i < totalCreated; i++) {
+            AssetId leaf = new AssetId(Uuids.timeBased());
+            allThirdLevelRelations.add(saveRelation(new EntityRelation(assetC, leaf, "thirdLevel")));
+        }
+
+        EntityRelationPathQuery query = new EntityRelationPathQuery(assetA, List.of(
+                new RelationPathLevel(EntitySearchDirection.TO, "firstLevel"),
+                new RelationPathLevel(EntitySearchDirection.TO, "secondLevel"),
+                new RelationPathLevel(EntitySearchDirection.FROM, "thirdLevel")
+        ));
+
+        // call a method that applies the default limit internally
+        List<EntityRelation> result = relationService.findByRelationPathQueryAsync(tenantId, query).get();
+
+        // verify that limit has been applied
+        assertThat(result).hasSize(limit);
+
+        // verify all returned are valid third-level relations under C
+        assertThat(result)
+                .allSatisfy(rel -> {
+                    assertThat(rel.getType()).isEqualTo("thirdLevel");
+                    assertThat(rel.getFrom()).isEqualTo(assetC);
+                });
+
+        // verify the returned subset is part of all created relations
+        assertThat(result).isSubsetOf(allThirdLevelRelations);
     }
 
     @Test