From 6cf314eaa01ce62d9ffbaba02e05d77a722817fc Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 19 Feb 2025 08:35:24 +0200 Subject: [PATCH 01/25] added NaN value when is not number in rolling --- .../server/service/cf/ctx/state/TsRollingArgumentEntry.java | 6 +++--- .../org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java | 3 +++ .../server/dao/service/CalculatedFieldServiceTest.java | 6 +++--- .../service/validator/CalculatedFieldDataValidatorTest.java | 3 +++ 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 866e2f5e09..da02b9be2e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -86,7 +86,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } @Override - public boolean updateEntry(ArgumentEntry entry) throws CalculatedFieldStateException { + public boolean updateEntry(ArgumentEntry entry) { if (entry instanceof TsRollingArgumentEntry tsRollingEntry) { updateTsRollingEntry(tsRollingEntry); } else if (entry instanceof SingleValueArgumentEntry singleValueEntry) { @@ -118,8 +118,8 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } cleanupExpiredRecords(); } catch (Exception e) { - log.warn("Time series rolling arguments supports only numeric values."); -// throw new IllegalArgumentException("Time series rolling arguments supports only numeric values."); + tsRecords.put(ts, Double.NaN); + log.warn("Invalid value '{}' for time series rolling arguments. Only numeric values are supported.", value.getValue()); } } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java index 843abf8d5c..c3a3606ad1 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java @@ -63,6 +63,9 @@ public class TbelCfTsRollingArg implements TbelCfArg, Iterable Date: Wed, 19 Feb 2025 08:59:42 +0200 Subject: [PATCH 02/25] fixed tests --- .../server/queue/discovery/HashPartitionServiceTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java index 52cbaa4add..3a015d23b2 100644 --- a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java @@ -426,6 +426,9 @@ public class HashPartitionServiceTest { topicService); ReflectionTestUtils.setField(partitionService, "coreTopic", "tb.core"); ReflectionTestUtils.setField(partitionService, "corePartitions", 10); + ReflectionTestUtils.setField(partitionService, "cfEventTopic", "tb_cf_event"); + ReflectionTestUtils.setField(partitionService, "cfStateTopic", "tb_cf_state"); + ReflectionTestUtils.setField(partitionService, "cfPartitions", 10); ReflectionTestUtils.setField(partitionService, "vcTopic", "tb.vc"); ReflectionTestUtils.setField(partitionService, "vcPartitions", 10); ReflectionTestUtils.setField(partitionService, "hashFunctionName", hashFunctionName); From 300650ceb6eaa2811db129be2069a23d59af721c Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 19 Feb 2025 09:06:23 +0200 Subject: [PATCH 03/25] fixed test where rolling arg is NaN --- .../service/cf/ctx/state/ScriptCalculatedFieldStateTest.java | 2 +- .../service/cf/ctx/state/TsRollingArgumentEntryTest.java | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index bc75bd74ff..e52e6e985a 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -151,7 +151,7 @@ public class ScriptCalculatedFieldStateTest { } private TsRollingArgumentEntry createRollingArgEntry() { - TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); + TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(5, 30000L); long ts = System.currentTimeMillis(); TreeMap values = new TreeMap<>(); diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java index 75aca61825..8f8abadf02 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java @@ -79,9 +79,8 @@ public class TsRollingArgumentEntryTest { void testUpdateEntryWhenValueIsNotNumber() { SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, new StringDataEntry("key", "string"), 123L); - assertThatThrownBy(() -> entry.updateEntry(newEntry)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Time series rolling arguments supports only numeric values."); + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords().get(ts - 10)).isNaN(); } @Test From 609a5dc34b67694620d8cc90620f85acf46ca9f8 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 19 Feb 2025 10:18:41 +0200 Subject: [PATCH 04/25] fixed cache onCfDelete and updated tenant profile properties --- ...CalculatedFieldEntityMessageProcessor.java | 29 ++++++++++--------- ...alculatedFieldManagerMessageProcessor.java | 1 + .../DefaultTenantProfileConfiguration.java | 3 +- .../CalculatedFieldDataValidator.java | 14 +++++++-- 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 7211ac0db9..7b7ce63d38 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -187,23 +187,23 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } } catch (Exception e) { + if (e instanceof CalculatedFieldException) { + throw (CalculatedFieldException) e; + } throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); } } - @SneakyThrows - private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) { + private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); } - @SneakyThrows - private void processAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) { + private void processAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); } - @SneakyThrows private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, - Map newArgValues, UUID tbMsgId, TbMsgType tbMsgType) { + Map newArgValues, UUID tbMsgId, TbMsgType tbMsgType) throws CalculatedFieldException { if (newArgValues.isEmpty()) { log.info("[{}] No new argument values to process for CF.", ctx.getCfId()); callback.onSuccess(CALLBACKS_PER_CF); @@ -241,15 +241,18 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return state; } - @SneakyThrows - private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) { + private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); if (state.isReady() && ctx.isInitialized()) { - CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); - state.checkStateSize(ctxId, ctx.getMaxStateSizeInKBytes()); - cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); - if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { - systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); + try { + CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); + state.checkStateSize(ctxId, ctx.getMaxStateSizeInKBytes()); + cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); + if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); + } + } catch (Exception e) { + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build(); } } else { state.checkStateSize(ctxId, ctx.getMaxStateSizeInKBytes()); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 5e005935aa..1d77f7cd76 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -308,6 +308,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.warn("[{}] CF was already deleted [{}]", tenantId, cfId); callback.onSuccess(); } else { + entityIdCalculatedFields.get(cfCtx.getEntityId()).remove(cfCtx); deleteLinks(cfCtx); EntityId entityId = cfCtx.getEntityId(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index 0139792fa7..0b7b030748 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -135,7 +135,7 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private double warnThreshold; - private long maxCalculatedFields; + private long maxCalculatedFieldsPerEntity; private long maxArgumentsPerCF; private long maxDataPointsPerRollingArg; private long maxStateSizeInKBytes; @@ -182,7 +182,6 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura case DASHBOARD -> maxDashboards; case RULE_CHAIN -> maxRuleChains; case EDGE -> maxEdges; - case CALCULATED_FIELD -> maxCalculatedFields; default -> 0; }; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index b52c2fe9b7..12c7e15f49 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -17,8 +17,8 @@ package org.thingsboard.server.dao.service.validator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.cf.CalculatedFieldDao; @@ -37,7 +37,7 @@ public class CalculatedFieldDataValidator extends DataValidator @Override protected void validateCreate(TenantId tenantId, CalculatedField calculatedField) { - validateNumberOfEntitiesPerTenant(tenantId, EntityType.CALCULATED_FIELD); + validateNumberOfCFsPerEntity(tenantId, calculatedField.getEntityId()); validateNumberOfArgumentsPerCF(tenantId, calculatedField); } @@ -51,6 +51,16 @@ public class CalculatedFieldDataValidator extends DataValidator return old; } + private void validateNumberOfCFsPerEntity(TenantId tenantId, EntityId entityId) { + long maxCFsPerEntity = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxCalculatedFieldsPerEntity); + if (maxCFsPerEntity <= 0) { + return; + } + if (calculatedFieldDao.countCFByEntityId(tenantId, entityId) >= maxCFsPerEntity) { + throw new DataValidationException("Calculated fields per entity limit reached!"); + } + } + private void validateNumberOfArgumentsPerCF(TenantId tenantId, CalculatedField calculatedField) { long maxArgumentsPerCF = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxArgumentsPerCF); if (maxArgumentsPerCF <= 0) { From e018282d0410349531a92422a23025d839b588bb Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 19 Feb 2025 17:51:48 +0200 Subject: [PATCH 05/25] minor fixes --- .../CalculatedFieldEntityMessageProcessor.java | 3 +++ .../CalculatedFieldManagerMessageProcessor.java | 13 +++---------- .../controller/CalculatedFieldControllerTest.java | 5 +++-- .../org/thingsboard/script/api/tbel/TbelCfArg.java | 2 ++ 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 7b7ce63d38..e551d9f067 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -116,6 +116,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM var cfState = getOrInitState(cfCtx); processStateIfReady(cfCtx, Collections.singletonList(cfCtx.getCfId()), cfState, null, null, msg.getCallback()); } catch (Exception e) { + if (e instanceof CalculatedFieldException cfe) { + throw cfe; + } throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(entityId).cause(e).build(); } } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 1d77f7cd76..b65a4072d3 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -30,7 +30,6 @@ import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; @@ -39,7 +38,6 @@ import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; @@ -53,11 +51,12 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; +import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; + /** * @author Andrew Shvayka @@ -358,7 +357,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var proto = msg.getProto(); var linksList = proto.getLinksList(); for (var linkProto : linksList) { - var link = toCalculatedFieldEntityCtxId(linkProto); + var link = fromProto(linkProto); var targetEntityId = link.entityId(); var targetEntityType = targetEntityId.getEntityType(); var cf = calculatedFields.get(link.cfId()); @@ -383,12 +382,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } } - private CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId(CalculatedFieldEntityCtxIdProto ctxIdProto) { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); - return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId); - } - private List filterCalculatedFieldLinks(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); var proto = msg.getProto(); diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index b1c7547251..79288fdb95 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -86,13 +86,14 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); assertThat(savedCalculatedField.getConfiguration()).isEqualTo(getCalculatedFieldConfig(testDevice.getId())); - assertThat(savedCalculatedField.getVersion()).isEqualTo(calculatedField.getVersion()); + assertThat(savedCalculatedField.getVersion()).isEqualTo(1L); savedCalculatedField.setName("Test CF"); CalculatedField updatedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); - assertThat(updatedCalculatedField).isEqualTo(savedCalculatedField); + assertThat(updatedCalculatedField.getName()).isEqualTo(savedCalculatedField.getName()); + assertThat(updatedCalculatedField.getVersion()).isEqualTo(savedCalculatedField.getVersion() + 1); doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) .andExpect(status().isOk()); diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java index 7fe75acd05..ddeb9d14c4 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java @@ -15,6 +15,7 @@ */ package org.thingsboard.script.api.tbel; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -29,6 +30,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; }) public interface TbelCfArg extends TbelCfObject { + @JsonIgnore String getType(); } From 40491f009e52d889ad6d501a34885e50e909a480 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 25 Feb 2025 09:43:24 +0200 Subject: [PATCH 06/25] removed cf debug events rate limit from tenant profile --- .../cf/CalculatedFieldIntegrationTest.java | 54 +++++++------------ .../server/common/data/limit/LimitedApi.java | 2 +- .../DefaultTenantProfileConfiguration.java | 1 - 3 files changed, 19 insertions(+), 38 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index f544f463fa..6e90d51d35 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -83,57 +83,52 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes calculatedField.setConfiguration(config); calculatedField.setVersion(1L); - // create CF -> perform initial calculation CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - await().atMost(5, TimeUnit.SECONDS) + await().alias("create CF -> perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); }); - // update telemetry -> recalculate state doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - await().atMost(5, TimeUnit.SECONDS) + await().alias("update telemetry -> recalculate state").atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); }); - // update CF output -> perform calculation with updated output Output savedOutput = savedCalculatedField.getConfiguration().getOutput(); savedOutput.setType(OutputType.ATTRIBUTES); savedOutput.setScope(AttributeScope.SERVER_SCOPE); savedOutput.setName("temperatureF"); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); - await().atMost(5, TimeUnit.SECONDS) + await().alias("update CF output -> perform calculation with updated output").atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); assertThat(temperatureF).isNotNull(); assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); }); - // update CF argument -> perform calculation with new argument Argument savedArgument = savedCalculatedField.getConfiguration().getArguments().get("T"); savedArgument.setRefEntityKey(new ReferencedEntityKey("deviceTemperature", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); - await().atMost(5, TimeUnit.SECONDS) + await().alias("update CF argument -> perform calculation with new argument").atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); assertThat(temperatureF).isNotNull(); assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); }); - // update CF expression -> perform calculation with new expression savedCalculatedField.getConfiguration().setExpression("1.8 * T + 32"); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); - await().atMost(5, TimeUnit.SECONDS) + await().alias("update CF expression -> perform calculation with new expression").atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); assertThat(temperatureF).isNotNull(); @@ -168,20 +163,18 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes calculatedField.setConfiguration(config); calculatedField.setVersion(1L); - // create CF -> state is not ready -> no calculation performed CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - await().atMost(5, TimeUnit.SECONDS) + await().alias("create CF -> state is not ready -> no calculation performed").atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); }); - // update telemetry -> perform calculation doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - await().atMost(5, TimeUnit.SECONDS) + await().alias("update telemetry -> perform calculation").atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); @@ -217,20 +210,18 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes calculatedField.setConfiguration(config); calculatedField.setVersion(1L); - // create CF -> perform initial calculation with default value CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - await().atMost(5, TimeUnit.SECONDS) + await().alias("create CF -> perform initial calculation with default value").atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("53.6"); }); - // update telemetry -> recalculate state doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - await().atMost(5, TimeUnit.SECONDS) + await().alias("update telemetry -> recalculate state").atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); @@ -283,10 +274,9 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes calculatedField.setConfiguration(config); calculatedField.setVersion(1L); - // create CF and perform initial calculation doPost("/api/calculatedField", calculatedField, CalculatedField.class); - await().atMost(5, TimeUnit.SECONDS) + await().alias("create CF and perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { // result of asset 1 ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); @@ -299,10 +289,9 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); }); - // update device telemetry -> recalculate state for all assets doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":25}")); - await().atMost(5, TimeUnit.SECONDS) + await().alias("update device telemetry -> recalculate state for all assets").atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { // result of asset 1 ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); @@ -315,10 +304,9 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); }); - // update asset 1 telemetry -> recalculate state only for asset 1 doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":15}")); - await().atMost(5, TimeUnit.SECONDS) + await().alias("update asset 1 telemetry -> recalculate state only for asset 1").atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { // result of asset 1 ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); @@ -331,10 +319,9 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); }); - // update asset 2 telemetry -> recalculate state only for asset 2 doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":5}")); - await().atMost(5, TimeUnit.SECONDS) + await().alias("update asset 2 telemetry -> recalculate state only for asset 2").atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { // result of asset 1 (no changes) ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); @@ -347,12 +334,11 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); }); - // add new entity to profile -> calculate state for new entity Asset asset3 = createAsset("Test asset 3", assetProfile.getId()); doPost("/api/plugins/telemetry/ASSET/" + asset3.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":13}")); Asset finalAsset3 = asset3; - await().atMost(5, TimeUnit.SECONDS) + await().alias("add new entity to profile -> calculate state for new entity").atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { // result of asset 3 ArrayNode z3 = getServerAttributes(finalAsset3.getId(), "z"); @@ -360,10 +346,9 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes assertThat(z3.get(0).get("value").asText()).isEqualTo("38.0"); }); - // update device telemetry -> recalculate state for all assets doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":20}")); - await().atMost(5, TimeUnit.SECONDS) + await().alias("update device telemetry -> recalculate state for all assets").atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { // result of asset 1 ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); @@ -386,11 +371,10 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes asset3.setAssetProfileId(newAssetProfile.getId()); asset3 = doPost("/api/asset", asset3, Asset.class); - // update device telemetry -> recalculate state for asset 1 and asset 2 doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":15}")); Asset updatedAsset3 = asset3; - await().atMost(5, TimeUnit.SECONDS) + await().alias("update device telemetry -> recalculate state for asset 1 and asset 2").atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { // result of asset 1 ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); @@ -438,20 +422,18 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes calculatedField.setConfiguration(config); calculatedField.setVersion(1L); - // create CF -> ctx is not initialized -> no calculation perform CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - await().atMost(5, TimeUnit.SECONDS) + await().alias("create CF -> ctx is not initialized -> no calculation perform").atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); }); - // update telemetry -> ctx is not initialized -> no calculation perform doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - await().atMost(5, TimeUnit.SECONDS) + await().alias("update telemetry -> ctx is not initialized -> no calculation perform").atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java b/common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java index 35068821ee..141a805dc0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java @@ -44,7 +44,7 @@ public enum LimitedApi { TRANSPORT_MESSAGES_PER_GATEWAY_DEVICE("transport messages per gateway device", false), EMAILS("emails sending", true), WS_SUBSCRIPTIONS("WS subscriptions", false), - CALCULATED_FIELD_DEBUG_EVENTS(DefaultTenantProfileConfiguration::getCalculatedFieldDebugEventsRateLimit, "calculated field debug events", true); + CALCULATED_FIELD_DEBUG_EVENTS("calculated field debug events", true); private Function configExtractor; @Getter diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index 0b7b030748..464ab410ab 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -140,7 +140,6 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private long maxDataPointsPerRollingArg; private long maxStateSizeInKBytes; private long maxSingleValueArgumentSizeInKBytes; - private String calculatedFieldDebugEventsRateLimit; @Override public long getProfileThreshold(ApiUsageRecordKey key) { From 6f1dd5a2d6d4041925e59c13df007afe9f042b24 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 25 Feb 2025 10:58:26 +0200 Subject: [PATCH 07/25] added new methods for working with rolling --- .../script/api/tbel/TbelCfTsRollingArg.java | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java index c3a3606ad1..ceae4e60c5 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Getter; +import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -60,6 +61,10 @@ public class TbelCfTsRollingArg implements TbelCfArg, Iterable val) { + min = val; + } + } + return min; + } + + public double mean() { + if (values.isEmpty()) { + return 0; + } + + double sum = sum(); + return Double.isNaN(sum) ? Double.NaN : sum / values.size(); + } + + public double std() { + if (values.isEmpty()) { + return 0; + } + + double mean = mean(); + if (Double.isNaN(mean)) { + return Double.NaN; + } + + double sum = 0; + for (TbelCfTsDoubleVal value : values) { + double val = value.getValue(); + if (Double.isNaN(val)) { + return Double.NaN; + } + sum += Math.pow(val - mean, 2); + } + return Math.sqrt(sum / values.size()); + } + + public double median() { + if (values.isEmpty()) { + return 0; + } + + List sortedValues = new ArrayList<>(); + for (TbelCfTsDoubleVal value : values) { + double val = value.getValue(); + if (Double.isNaN(val)) { + return Double.NaN; + } + sortedValues.add(val); + } + Collections.sort(sortedValues); + + int size = sortedValues.size(); + return (size % 2 == 1) + ? sortedValues.get(size / 2) + : (sortedValues.get(size / 2 - 1) + sortedValues.get(size / 2)) / 2.0; + } + + public double count() { + return hasNaN() ? Double.NaN : values.size(); + } + + public double last() { + if (values.isEmpty()) { + return 0; + } + + return hasNaN() ? Double.NaN : values.get(values.size() - 1).getValue(); + } + + public double first() { + if (values.isEmpty()) { + return 0; + } + + return hasNaN() ? Double.NaN : values.get(0).getValue(); + } + + public double sum() { + if (values.isEmpty()) { + return 0; + } + + double sum = 0; + for (TbelCfTsDoubleVal value : values) { + double val = value.getValue(); + if (Double.isNaN(val)) { + return Double.NaN; + } + sum += val; + } + return sum; + } + + private boolean hasNaN() { + for (TbelCfTsDoubleVal value : values) { + if (Double.isNaN(value.getValue())) { + return true; + } + } + return false; + } + @JsonIgnore public int getSize() { return values.size(); From 6698a8f8f5ae530de6299f9c76b9ba5ddfca5cc1 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 25 Feb 2025 11:33:56 +0200 Subject: [PATCH 08/25] fixed tenant actor test --- .../org/thingsboard/server/actors/tenant/TenantActorTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/actors/tenant/TenantActorTest.java b/application/src/test/java/org/thingsboard/server/actors/tenant/TenantActorTest.java index cad933ccc5..ed8468aa6a 100644 --- a/application/src/test/java/org/thingsboard/server/actors/tenant/TenantActorTest.java +++ b/application/src/test/java/org/thingsboard/server/actors/tenant/TenantActorTest.java @@ -27,6 +27,7 @@ import org.thingsboard.server.actors.TbActorSystemSettings; import org.thingsboard.server.actors.TbEntityActorId; import org.thingsboard.server.actors.ruleChain.RuleChainActor; import org.thingsboard.server.actors.ruleChain.RuleChainToRuleChainMsg; +import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.RuleChainErrorActor; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.id.DeviceId; @@ -116,6 +117,7 @@ public class TenantActorTest { TbActorSystemSettings settings = new TbActorSystemSettings(0, 0, 0); TbActorSystem system = spy(new DefaultTbActorSystem(settings)); system.createDispatcher(RULE_DISPATCHER_NAME, mock()); + system.createDispatcher(DefaultActorService.CF_MANAGER_DISPATCHER_NAME, mock()); TbActorMailbox tenantCtx = new TbActorMailbox(system, settings, null, mock(), mock(), null); tenantActor.init(tenantCtx); From a6a29513f618a826aa7c62cb0f46ca6954e24f44 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 25 Feb 2025 11:50:34 +0200 Subject: [PATCH 09/25] removed broadcast asset updates event to transport --- .../server/service/queue/DefaultTbClusterService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 920a7563dc..19c386fff3 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -697,7 +697,6 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onAssetUpdated(Asset entity, Asset old) { var created = old == null; - broadcastEntityChangeToTransport(entity.getTenantId(), entity.getId(), entity, null); if (old != null) { boolean assetTypeChanged = !entity.getAssetProfileId().equals(old.getAssetProfileId()); if (assetTypeChanged) { From f362749a7b24c5f9c32fab7a38ea0cb97db8fc0a Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 25 Feb 2025 12:09:25 +0200 Subject: [PATCH 10/25] fixed tenant profile controller test --- .../server/controller/TenantProfileControllerTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java index 989e80e6e5..ad8eb188c9 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.queue.TbQueueCallback; import java.util.ArrayList; import java.util.Collections; @@ -44,6 +45,7 @@ import java.util.List; import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -354,7 +356,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { argument -> argument.getClass().equals(TenantProfile.class); if (ComponentLifecycleEvent.DELETED.equals(event)) { Mockito.verify(tbClusterService, times(cntTime)).onTenantProfileDelete(Mockito.argThat(matcherTenantProfile), - Mockito.isNull()); + eq(TbQueueCallback.EMPTY)); testBroadcastEntityStateChangeEventNever(createEntityId_NULL_UUID(new Tenant())); } else { Mockito.verify(tbClusterService, times(cntTime)).onTenantProfileChange(Mockito.argThat(matcherTenantProfile), From a52c578ecdad303a8bdd2b8a170d77e69facb2fd Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 25 Feb 2025 15:06:40 +0200 Subject: [PATCH 11/25] minor fixes --- .../CalculatedFieldEntityMessageProcessor.java | 4 ++-- .../server/service/cf/ctx/state/TsRollingArgumentEntry.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index ecf1b140f2..e564489da2 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -194,8 +194,8 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } } catch (Exception e) { - if (e instanceof CalculatedFieldException) { - throw (CalculatedFieldException) e; + if (e instanceof CalculatedFieldException cfe) { + throw cfe; } throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index da02b9be2e..90068d6b6f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -119,7 +119,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { cleanupExpiredRecords(); } catch (Exception e) { tsRecords.put(ts, Double.NaN); - log.warn("Invalid value '{}' for time series rolling arguments. Only numeric values are supported.", value.getValue()); + log.debug("Invalid value '{}' for time series rolling arguments. Only numeric values are supported.", value.getValue()); } } From 49328d270bf7e7236f7235ddf933ec4a926bc68a Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 25 Feb 2025 17:13:05 +0200 Subject: [PATCH 12/25] wip ts/attributes deletion --- ...CalculatedFieldEntityMessageProcessor.java | 75 +++++++++++++++++-- .../cf/CalculatedFieldQueueService.java | 6 ++ .../DefaultCalculatedFieldQueueService.java | 37 +++++++++ .../cf/ctx/state/CalculatedFieldCtx.java | 48 +++++++++++- .../DefaultTelemetrySubscriptionService.java | 23 +++++- common/proto/src/main/proto/queue.proto | 8 +- .../engine/api/AttributesDeleteRequest.java | 21 +++++- .../engine/api/TimeseriesDeleteRequest.java | 21 +++++- .../telemetry/TbMsgDeleteAttributesNode.java | 2 + 9 files changed, 223 insertions(+), 18 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 2da53757ff..a53e77d241 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -24,11 +24,13 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -164,7 +166,11 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (cfIds.contains(ctx.getCfId())) { callback.onSuccess(CALLBACKS_PER_CF); } else { - if (proto.getTsDataCount() > 0) { + if (proto.getRemovedTsKeysCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapDeletedAttributesToDefault(ctx, msg.getEntityId(), proto.getScope(), proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + } else if (proto.getRemovedAttrKeysCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapDeletedAttributesToDefault(ctx, msg.getEntityId(), proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + } else if (proto.getTsDataCount() > 0) { processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); } else if (proto.getAttrDataCount() > 0) { processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); @@ -182,7 +188,11 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (cfIds.contains(ctx.getCfId())) { callback.onSuccess(CALLBACKS_PER_CF); } else { - if (proto.getTsDataCount() > 0) { + if (proto.getRemovedTsKeysCount() > 0) { + processRemovedTelemetry(ctx, proto, cfIdList, callback); + } else if (proto.getRemovedAttrKeysCount() > 0) { + processRemovedAttributes(ctx, proto, cfIdList, callback); + } else if (proto.getTsDataCount() > 0) { processTelemetry(ctx, proto, cfIdList, callback); } else if (proto.getAttrDataCount() > 0) { processAttributes(ctx, proto, cfIdList, callback); @@ -191,23 +201,43 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } } catch (Exception e) { + if (e instanceof CalculatedFieldException) { + throw (CalculatedFieldException) e; + } throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); } } - @SneakyThrows - private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) { + private void processRemovedTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + // reinit cf (consider fetching only removed ts) + log.info("Force reinitialization of CF: [{}].", ctx.getCfId()); + states.remove(ctx.getCfId()); + try { + var state = getOrInitState(ctx); + if (state.isSizeOk()) { + processStateIfReady(ctx, Collections.singletonList(ctx.getCfId()), state, null, null, callback); + } else { + throw new RuntimeException(ctx.getSizeExceedsLimitMessage()); + } + } catch (Exception e) { + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); + } + } + + private void processRemovedAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + processArgumentValuesUpdate(ctx, cfIdList, callback, mapDeletedAttributesToDefault(ctx, proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + } + + private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); } - @SneakyThrows - private void processAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) { + private void processAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); } - @SneakyThrows private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, - Map newArgValues, UUID tbMsgId, TbMsgType tbMsgType) { + Map newArgValues, UUID tbMsgId, TbMsgType tbMsgType) throws CalculatedFieldException { if (newArgValues.isEmpty()) { log.info("[{}] No new argument values to process for CF.", ctx.getCfId()); callback.onSuccess(CALLBACKS_PER_CF); @@ -348,6 +378,35 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return arguments; } + private Map mapDeletedAttributesToDefault(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List removedAttrKeys) { + var argNames = ctx.getLinkedEntityArguments().get(entityId); + if (argNames.isEmpty()) { + return Collections.emptyMap(); + } + return mapToArgumentsDefaultValue(argNames, ctx.getArguments(), scope, removedAttrKeys); + } + + private Map mapDeletedAttributesToDefault(CalculatedFieldCtx ctx, AttributeScopeProto scope, List removedAttrKeys) { + return mapToArgumentsDefaultValue(ctx.getMainEntityArguments(), ctx.getArguments(), scope, removedAttrKeys); + } + + private static Map mapToArgumentsDefaultValue(Map argNames, Map configArguments, AttributeScopeProto scope, List removedAttrKeys) { + Map arguments = new HashMap<>(); + for (String removedKey : removedAttrKeys) { + ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + String argName = argNames.get(key); + if (argName != null) { + Argument argument = configArguments.get(argName); + String defaultValue = (argument != null) ? argument.getDefaultValue() : null; + arguments.put(argName, (defaultValue != null) + ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) + : new SingleValueArgumentEntry()); + + } + } + return arguments; + } + private static List getCalculatedFieldIds(CalculatedFieldTelemetryMsgProto proto) { List cfIds = new LinkedList<>(); for (var cfId : proto.getPreviousCalculatedFieldsList()) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java index b84b54af81..1b0d1462f2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java @@ -16,7 +16,9 @@ package org.thingsboard.server.service.cf; import com.google.common.util.concurrent.FutureCallback; +import org.thingsboard.rule.engine.api.AttributesDeleteRequest; import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; @@ -34,4 +36,8 @@ public interface CalculatedFieldQueueService { void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); + void pushRequestToQueue(AttributesDeleteRequest request, List result, FutureCallback callback); + + void pushRequestToQueue(TimeseriesDeleteRequest request, List result, FutureCallback callback); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 9d1f9b6db5..13905dbcce 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -21,7 +21,9 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.AttributesDeleteRequest; import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; @@ -48,6 +50,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import java.util.ArrayList; import java.util.EnumSet; import java.util.List; import java.util.Set; @@ -104,6 +107,23 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } + @Override + public void pushRequestToQueue(AttributesDeleteRequest request, List result, FutureCallback callback) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matchesKeys(result, request.getScope()), cf -> cf.linkMatchesAttrKeys(entityId, result, request.getScope()), + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); + } + + @Override + public void pushRequestToQueue(TimeseriesDeleteRequest request, List result, FutureCallback callback) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matchesKeys(result), cf -> cf.linkMatchesTsKeys(entityId, result), + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); + } + private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, Predicate mainEntityFilter, Predicate linkedEntityFilter, Supplier msg, FutureCallback callback) { @@ -174,6 +194,23 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS return msg.build(); } + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesDeleteRequest request, List removedKeys) { + CalculatedFieldTelemetryMsgProto telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), new ArrayList<>(), request.getTbMsgId(), request.getTbMsgType()) + .setScope(AttributeScopeProto.valueOf(request.getScope().name())) + .addAllRemovedAttrKeys(removedKeys).build(); + return ToCalculatedFieldMsg.newBuilder() + .setTelemetryMsg(telemetryMsg) + .build(); + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesDeleteRequest request, List removedKeys) { + CalculatedFieldTelemetryMsgProto telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), new ArrayList<>(), request.getTbMsgId(), request.getTbMsgType()) + .addAllRemovedTsKeys(removedKeys).build(); + return ToCalculatedFieldMsg.newBuilder() + .setTelemetryMsg(telemetryMsg) + .build(); + } + private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List calculatedFieldIds, UUID tbMsgId, TbMsgType tbMsgType) { CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = CalculatedFieldTelemetryMsgProto.newBuilder(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 4e4844dbe8..e254680019 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -151,10 +151,28 @@ public class CalculatedFieldCtx { } } + public boolean matchesKeys(List keys, AttributeScope scope) { + return matchesAttributesKeys(mainEntityArguments, keys, scope); + } + + public boolean matchesKeys(List keys) { + return matchesTimeSeriesKeys(mainEntityArguments, keys); + } + public boolean matches(List values, AttributeScope scope) { return matchesAttributes(mainEntityArguments, values, scope); } + public boolean linkMatchesAttrKeys(EntityId entityId, List keys, AttributeScope scope) { + var map = linkedEntityArguments.get(entityId); + return map != null && matchesAttributesKeys(map, keys, scope); + } + + public boolean linkMatchesTsKeys(EntityId entityId, List keys) { + var map = linkedEntityArguments.get(entityId); + return map != null && matchesTimeSeriesKeys(map, keys); + } + public boolean linkMatches(EntityId entityId, List values, AttributeScope scope) { var map = linkedEntityArguments.get(entityId); return map != null && matchesAttributes(map, values, scope); @@ -179,6 +197,16 @@ public class CalculatedFieldCtx { return false; } + private boolean matchesAttributesKeys(Map argMap, List keys, AttributeScope scope) { + for (String key : keys) { + ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); + if (argMap.containsKey(attrKey)) { + return true; + } + } + return false; + } + private boolean matchesTimeSeries(Map argMap, List values) { for (TsKvEntry tsKv : values) { ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_LATEST, null); @@ -193,8 +221,26 @@ public class CalculatedFieldCtx { return false; } + private boolean matchesTimeSeriesKeys(Map argMap, List keys) { + for (String key : keys) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); + if (argMap.containsKey(latestKey)) { + return true; + } + ReferencedEntityKey rollingKey = new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null); + if (argMap.containsKey(rollingKey)) { + return true; + } + } + return false; + } + public boolean linkMatches(EntityId entityId, CalculatedFieldTelemetryMsgProto proto) { - if (!proto.getTsDataList().isEmpty()) { + if (!proto.getRemovedTsKeysList().isEmpty()) { + return linkMatchesTsKeys(entityId, proto.getRemovedTsKeysList()); + } else if (!proto.getRemovedAttrKeysList().isEmpty()) { + return linkMatchesAttrKeys(entityId, proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); + } else if (!proto.getTsDataList().isEmpty()) { List updatedTelemetry = proto.getTsDataList().stream() .map(ProtoUtils::fromProto) .toList(); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 79222ca002..3f9b88b8d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -186,7 +186,9 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void deleteAttributesInternal(AttributesDeleteRequest request) { ListenableFuture> deleteFuture = attrService.removeAll(request.getTenantId(), request.getEntityId(), request.getScope(), request.getKeys()); - addMainCallback(deleteFuture, request.getCallback()); + DonAsynchron.withCallback(deleteFuture, result -> { + calculatedFieldQueueService.pushRequestToQueue(request, result, request.getCallback()); + }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(deleteFuture, success -> onAttributesDelete(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getKeys(), request.isNotifyDevice())); } @@ -206,7 +208,10 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer deleteFuture = tsService.remove(request.getTenantId(), request.getEntityId(), request.getDeleteHistoryQueries()); addWsCallback(deleteFuture, result -> onTimeSeriesDelete(request.getTenantId(), request.getEntityId(), request.getKeys(), result)); } - addMainCallback(deleteFuture, __ -> request.getCallback().onSuccess(request.getKeys()), request.getCallback()::onFailure); + DonAsynchron.withCallback(deleteFuture, result -> { + calculatedFieldQueueService.pushRequestToQueue(request, request.getKeys(), getCalculatedFieldCallback(request.getCallback(), request.getKeys())); + }, safeCallback(getCalculatedFieldCallback(request.getCallback(), request.getKeys())), tsCallBackExecutor); +// addMainCallback(deleteFuture, __ -> request.getCallback().onSuccess(request.getKeys()), request.getCallback()::onFailure); } else { ListenableFuture> deleteFuture = tsService.removeAllLatest(request.getTenantId(), request.getEntityId()); addMainCallback(deleteFuture, request.getCallback()::onSuccess, request.getCallback()::onFailure); @@ -338,4 +343,18 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer }; } + private FutureCallback getCalculatedFieldCallback(FutureCallback> originalCallback, List keys) { + return new FutureCallback() { + @Override + public void onSuccess(Void unused) { + originalCallback.onSuccess(keys); + } + + @Override + public void onFailure(Throwable t) { + originalCallback.onFailure(t); + } + }; + } + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 5fc7e7c59c..432edd2201 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -783,9 +783,11 @@ message CalculatedFieldTelemetryMsgProto { repeated TsKvProto tsData = 9; AttributeScopeProto scope = 10; repeated AttributeValueProto attrData = 11; - int64 tbMsgIdMSB = 12; - int64 tbMsgIdLSB = 13; - string tbMsgType = 14; + repeated string removedTsKeys = 12; + repeated string removedAttrKeys = 13; + int64 tbMsgIdMSB = 14; + int64 tbMsgIdLSB = 15; + string tbMsgType = 16; } message CalculatedFieldLinkedTelemetryMsgProto { diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesDeleteRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesDeleteRequest.java index 118c62e78c..e161c0f165 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesDeleteRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesDeleteRequest.java @@ -24,8 +24,10 @@ import lombok.ToString; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; import java.util.List; +import java.util.UUID; @Getter @ToString @@ -37,6 +39,8 @@ public class AttributesDeleteRequest { private final AttributeScope scope; private final List keys; private final boolean notifyDevice; + private final UUID tbMsgId; + private final TbMsgType tbMsgType; private final FutureCallback callback; public static Builder builder() { @@ -50,9 +54,12 @@ public class AttributesDeleteRequest { private AttributeScope scope; private List keys; private boolean notifyDevice; + private UUID tbMsgId; + private TbMsgType tbMsgType; private FutureCallback callback; - Builder() {} + Builder() { + } public Builder tenantId(TenantId tenantId) { this.tenantId = tenantId; @@ -89,6 +96,16 @@ public class AttributesDeleteRequest { return this; } + public Builder tbMsgId(UUID tbMsgId) { + this.tbMsgId = tbMsgId; + return this; + } + + public Builder tbMsgType(TbMsgType tbMsgType) { + this.tbMsgType = tbMsgType; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -109,7 +126,7 @@ public class AttributesDeleteRequest { } public AttributesDeleteRequest build() { - return new AttributesDeleteRequest(tenantId, entityId, scope, keys, notifyDevice, callback); + return new AttributesDeleteRequest(tenantId, entityId, scope, keys, notifyDevice, tbMsgId, tbMsgType, callback); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java index b124806fff..62fc28737c 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java @@ -23,8 +23,10 @@ import lombok.ToString; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.msg.TbMsgType; import java.util.List; +import java.util.UUID; @Getter @ToString @@ -35,6 +37,8 @@ public class TimeseriesDeleteRequest { private final EntityId entityId; private final List keys; private final List deleteHistoryQueries; + private final UUID tbMsgId; + private final TbMsgType tbMsgType; private final FutureCallback> callback; public static Builder builder() { @@ -47,9 +51,12 @@ public class TimeseriesDeleteRequest { private EntityId entityId; private List keys; private List deleteHistoryQueries; + private UUID tbMsgId; + private TbMsgType tbMsgType; private FutureCallback> callback; - Builder() {} + Builder() { + } public Builder tenantId(TenantId tenantId) { this.tenantId = tenantId; @@ -71,13 +78,23 @@ public class TimeseriesDeleteRequest { return this; } + public Builder tbMsgId(UUID tbMsgId) { + this.tbMsgId = tbMsgId; + return this; + } + + public Builder tbMsgType(TbMsgType tbMsgType) { + this.tbMsgType = tbMsgType; + return this; + } + public Builder callback(FutureCallback> callback) { this.callback = callback; return this; } public TimeseriesDeleteRequest build() { - return new TimeseriesDeleteRequest(tenantId, entityId, keys, deleteHistoryQueries, callback); + return new TimeseriesDeleteRequest(tenantId, entityId, keys, deleteHistoryQueries, tbMsgId, tbMsgType, callback); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNode.java index a89a4f37d8..87a8b73c49 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNode.java @@ -76,6 +76,8 @@ public class TbMsgDeleteAttributesNode implements TbNode { .scope(scope) .keys(keysToDelete) .notifyDevice(checkNotifyDevice(msg.getMetaData().getValue(NOTIFY_DEVICE_METADATA_KEY), scope)) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) .callback(config.isSendAttributesDeletedNotification() ? new AttributesDeleteNodeCallback(ctx, msg, scope.name(), keysToDelete) : new TelemetryNodeCallback(ctx, msg)) From dd0b73cfb3d9ac56db83c98a660741a50fd4ec3b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 26 Feb 2025 11:38:20 +0200 Subject: [PATCH 13/25] added implementation for ts deletion --- ...CalculatedFieldEntityMessageProcessor.java | 77 ++++++++++--------- .../cf/CalculatedFieldProcessingService.java | 5 ++ ...faultCalculatedFieldProcessingService.java | 22 ++++++ .../service/cf/ctx/state/ArgumentEntry.java | 4 + .../ctx/state/BaseCalculatedFieldState.java | 2 +- .../cf/ctx/state/CalculatedFieldCtx.java | 70 ++++++++--------- .../ctx/state/SingleValueArgumentEntry.java | 8 ++ .../cf/ctx/state/TsRollingArgumentEntry.java | 8 ++ .../DefaultTelemetrySubscriptionService.java | 5 +- 9 files changed, 126 insertions(+), 75 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index a53e77d241..adc4615faa 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -24,6 +24,7 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; @@ -58,6 +59,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** @@ -166,14 +168,15 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (cfIds.contains(ctx.getCfId())) { callback.onSuccess(CALLBACKS_PER_CF); } else { - if (proto.getRemovedTsKeysCount() > 0) { - processArgumentValuesUpdate(ctx, cfIds, callback, mapDeletedAttributesToDefault(ctx, msg.getEntityId(), proto.getScope(), proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); - } else if (proto.getRemovedAttrKeysCount() > 0) { - processArgumentValuesUpdate(ctx, cfIds, callback, mapDeletedAttributesToDefault(ctx, msg.getEntityId(), proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); - } else if (proto.getTsDataCount() > 0) { + if (proto.getTsDataCount() > 0) { processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); } else if (proto.getAttrDataCount() > 0) { processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); + } + if (proto.getRemovedTsKeysCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + } else if (proto.getRemovedAttrKeysCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithDefaultValue(ctx, msg.getEntityId(), proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); } else { callback.onSuccess(CALLBACKS_PER_CF); } @@ -188,14 +191,15 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (cfIds.contains(ctx.getCfId())) { callback.onSuccess(CALLBACKS_PER_CF); } else { + if (proto.getTsDataCount() > 0) { + processTelemetry(ctx, proto, cfIdList, callback); + } else if (proto.getAttrDataCount() > 0) { + processAttributes(ctx, proto, cfIdList, callback); + } if (proto.getRemovedTsKeysCount() > 0) { processRemovedTelemetry(ctx, proto, cfIdList, callback); } else if (proto.getRemovedAttrKeysCount() > 0) { processRemovedAttributes(ctx, proto, cfIdList, callback); - } else if (proto.getTsDataCount() > 0) { - processTelemetry(ctx, proto, cfIdList, callback); - } else if (proto.getAttrDataCount() > 0) { - processAttributes(ctx, proto, cfIdList, callback); } else { callback.onSuccess(CALLBACKS_PER_CF); } @@ -208,26 +212,6 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } - private void processRemovedTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { - // reinit cf (consider fetching only removed ts) - log.info("Force reinitialization of CF: [{}].", ctx.getCfId()); - states.remove(ctx.getCfId()); - try { - var state = getOrInitState(ctx); - if (state.isSizeOk()) { - processStateIfReady(ctx, Collections.singletonList(ctx.getCfId()), state, null, null, callback); - } else { - throw new RuntimeException(ctx.getSizeExceedsLimitMessage()); - } - } catch (Exception e) { - throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); - } - } - - private void processRemovedAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { - processArgumentValuesUpdate(ctx, cfIdList, callback, mapDeletedAttributesToDefault(ctx, proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); - } - private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); } @@ -236,6 +220,14 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); } + private void processRemovedTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + } + + private void processRemovedAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithDefaultValue(ctx, proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + } + private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, Map newArgValues, UUID tbMsgId, TbMsgType tbMsgType) throws CalculatedFieldException { if (newArgValues.isEmpty()) { @@ -334,7 +326,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return mapToArguments(argNames, data); } - private static Map mapToArguments(Map argNames, List data) { + private Map mapToArguments(Map argNames, List data) { if (argNames.isEmpty()) { return Collections.emptyMap(); } @@ -366,7 +358,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return mapToArguments(argNames, scope, attrDataList); } - private static Map mapToArguments(Map argNames, AttributeScopeProto scope, List attrDataList) { + private Map mapToArguments(Map argNames, AttributeScopeProto scope, List attrDataList) { Map arguments = new HashMap<>(); for (AttributeValueProto item : attrDataList) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); @@ -378,19 +370,19 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return arguments; } - private Map mapDeletedAttributesToDefault(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List removedAttrKeys) { + private Map mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List removedAttrKeys) { var argNames = ctx.getLinkedEntityArguments().get(entityId); if (argNames.isEmpty()) { return Collections.emptyMap(); } - return mapToArgumentsDefaultValue(argNames, ctx.getArguments(), scope, removedAttrKeys); + return mapToArgumentsWithDefaultValue(argNames, ctx.getArguments(), scope, removedAttrKeys); } - private Map mapDeletedAttributesToDefault(CalculatedFieldCtx ctx, AttributeScopeProto scope, List removedAttrKeys) { - return mapToArgumentsDefaultValue(ctx.getMainEntityArguments(), ctx.getArguments(), scope, removedAttrKeys); + private Map mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, AttributeScopeProto scope, List removedAttrKeys) { + return mapToArgumentsWithDefaultValue(ctx.getMainEntityArguments(), ctx.getArguments(), scope, removedAttrKeys); } - private static Map mapToArgumentsDefaultValue(Map argNames, Map configArguments, AttributeScopeProto scope, List removedAttrKeys) { + private Map mapToArgumentsWithDefaultValue(Map argNames, Map configArguments, AttributeScopeProto scope, List removedAttrKeys) { Map arguments = new HashMap<>(); for (String removedKey : removedAttrKeys) { ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); @@ -398,7 +390,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (argName != null) { Argument argument = configArguments.get(argName); String defaultValue = (argument != null) ? argument.getDefaultValue() : null; - arguments.put(argName, (defaultValue != null) + arguments.put(argName, StringUtils.isNotEmpty(defaultValue) ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) : new SingleValueArgumentEntry()); @@ -407,6 +399,17 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return arguments; } + private Map mapToArgumentsWithFetchedValue(CalculatedFieldCtx ctx, List removedTelemetryKeys) { + Map deletedArguments = ctx.getArguments().entrySet().stream() + .filter(entry -> removedTelemetryKeys.contains(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); + + fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); + return fetchedArgs; + } + private static List getCalculatedFieldIds(CalculatedFieldTelemetryMsgProto proto) { List cfIds = new LinkedList<>(); for (var cfId : proto.getPreviousCalculatedFieldsList()) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index 24b428593a..c81690110c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -17,20 +17,25 @@ package org.thingsboard.server.service.cf; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import java.util.List; +import java.util.Map; public interface CalculatedFieldProcessingService { ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); + Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments); + void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index 3abaea75b4..a98566b48c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -147,6 +147,28 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP }, calculatedFieldCallbackExecutor); } + @Override + public Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments) { + Map> argFutures = new HashMap<>(); + for (var entry : arguments.entrySet()) { + var argEntityId = entry.getValue().getRefEntityId() != null ? entry.getValue().getRefEntityId() : entityId; + var argValueFuture = fetchKvEntry(tenantId, argEntityId, entry.getValue()); + argFutures.put(entry.getKey(), argValueFuture); + } + return argFutures.entrySet().stream() + .collect(Collectors.toMap( + Entry::getKey, // Keep the key as is + entry -> { + try { + // Resolve the future to get the value + return entry.getValue().get(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException("Error getting future result for key: " + entry.getKey(), e); + } + } + )); + } + @Override public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculatedFieldResult, List cfIds, TbCallback callback) { try { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 13656d5b0a..edbcb456c6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -54,4 +54,8 @@ public interface ArgumentEntry { TbelCfArg toTbelCfArg(); + boolean isForceResetPrevious(); + + void setForceResetPrevious(boolean forceResetPrevious); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 0f77027650..e889fdb487 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -55,7 +55,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { ArgumentEntry newEntry = entry.getValue(); ArgumentEntry existingEntry = arguments.get(key); - if (existingEntry == null) { + if (existingEntry == null || newEntry.isForceResetPrevious()) { validateNewEntry(newEntry); arguments.put(key, newEntry); stateUpdated = true; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index e254680019..8cca5c3961 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -151,28 +151,10 @@ public class CalculatedFieldCtx { } } - public boolean matchesKeys(List keys, AttributeScope scope) { - return matchesAttributesKeys(mainEntityArguments, keys, scope); - } - - public boolean matchesKeys(List keys) { - return matchesTimeSeriesKeys(mainEntityArguments, keys); - } - public boolean matches(List values, AttributeScope scope) { return matchesAttributes(mainEntityArguments, values, scope); } - public boolean linkMatchesAttrKeys(EntityId entityId, List keys, AttributeScope scope) { - var map = linkedEntityArguments.get(entityId); - return map != null && matchesAttributesKeys(map, keys, scope); - } - - public boolean linkMatchesTsKeys(EntityId entityId, List keys) { - var map = linkedEntityArguments.get(entityId); - return map != null && matchesTimeSeriesKeys(map, keys); - } - public boolean linkMatches(EntityId entityId, List values, AttributeScope scope) { var map = linkedEntityArguments.get(entityId); return map != null && matchesAttributes(map, values, scope); @@ -187,7 +169,7 @@ public class CalculatedFieldCtx { return map != null && matchesTimeSeries(map, values); } - private static boolean matchesAttributes(Map argMap, List values, AttributeScope scope) { + private boolean matchesAttributes(Map argMap, List values, AttributeScope scope) { for (AttributeKvEntry attrKv : values) { ReferencedEntityKey attrKey = new ReferencedEntityKey(attrKv.getKey(), ArgumentType.ATTRIBUTE, scope); if (argMap.containsKey(attrKey)) { @@ -197,16 +179,6 @@ public class CalculatedFieldCtx { return false; } - private boolean matchesAttributesKeys(Map argMap, List keys, AttributeScope scope) { - for (String key : keys) { - ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); - if (argMap.containsKey(attrKey)) { - return true; - } - } - return false; - } - private boolean matchesTimeSeries(Map argMap, List values) { for (TsKvEntry tsKv : values) { ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_LATEST, null); @@ -221,6 +193,24 @@ public class CalculatedFieldCtx { return false; } + public boolean matchesKeys(List keys, AttributeScope scope) { + return matchesAttributesKeys(mainEntityArguments, keys, scope); + } + + public boolean matchesKeys(List keys) { + return matchesTimeSeriesKeys(mainEntityArguments, keys); + } + + private boolean matchesAttributesKeys(Map argMap, List keys, AttributeScope scope) { + for (String key : keys) { + ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); + if (argMap.containsKey(attrKey)) { + return true; + } + } + return false; + } + private boolean matchesTimeSeriesKeys(Map argMap, List keys) { for (String key : keys) { ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); @@ -235,22 +225,32 @@ public class CalculatedFieldCtx { return false; } + public boolean linkMatchesAttrKeys(EntityId entityId, List keys, AttributeScope scope) { + var map = linkedEntityArguments.get(entityId); + return map != null && matchesAttributesKeys(map, keys, scope); + } + + public boolean linkMatchesTsKeys(EntityId entityId, List keys) { + var map = linkedEntityArguments.get(entityId); + return map != null && matchesTimeSeriesKeys(map, keys); + } + public boolean linkMatches(EntityId entityId, CalculatedFieldTelemetryMsgProto proto) { - if (!proto.getRemovedTsKeysList().isEmpty()) { - return linkMatchesTsKeys(entityId, proto.getRemovedTsKeysList()); - } else if (!proto.getRemovedAttrKeysList().isEmpty()) { - return linkMatchesAttrKeys(entityId, proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); - } else if (!proto.getTsDataList().isEmpty()) { + if (!proto.getTsDataList().isEmpty()) { List updatedTelemetry = proto.getTsDataList().stream() .map(ProtoUtils::fromProto) .toList(); return linkMatches(entityId, updatedTelemetry); - } else { + } else if (!proto.getAttrDataList().isEmpty()) { AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); List updatedTelemetry = proto.getAttrDataList().stream() .map(ProtoUtils::fromProto) .toList(); return linkMatches(entityId, updatedTelemetry, scope); + } else if (!proto.getRemovedTsKeysList().isEmpty()) { + return linkMatchesTsKeys(entityId, proto.getRemovedTsKeysList()); + } else { + return linkMatchesAttrKeys(entityId, proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index c096233224..031629993d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -38,6 +38,8 @@ public class SingleValueArgumentEntry implements ArgumentEntry { private BasicKvEntry kvEntryValue; private Long version; + private boolean forceResetPrevious; + public SingleValueArgumentEntry(TsKvProto entry) { this.ts = entry.getTs(); this.version = entry.getVersion(); @@ -61,6 +63,12 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.kvEntryValue = ProtoUtils.basicKvEntryFromKvEntry(entry); } + public SingleValueArgumentEntry(long ts, BasicKvEntry kvEntryValue, Long version) { + this.ts = ts; + this.kvEntryValue = kvEntryValue; + this.version = version; + } + @Override public ArgumentEntryType getType() { return ArgumentEntryType.SINGLE_VALUE; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 866e2f5e09..b10e9f4c7a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -42,6 +42,8 @@ public class TsRollingArgumentEntry implements ArgumentEntry { private Long timeWindow; private TreeMap tsRecords = new TreeMap<>(); + private boolean forceResetPrevious; + public TsRollingArgumentEntry(List kvEntries, int limit, long timeWindow) { this.limit = limit; this.timeWindow = timeWindow; @@ -60,6 +62,12 @@ public class TsRollingArgumentEntry implements ArgumentEntry { this.timeWindow = timeWindow; } + public TsRollingArgumentEntry(Integer limit, Long timeWindow, TreeMap tsRecords) { + this.limit = limit; + this.timeWindow = timeWindow; + this.tsRecords = tsRecords; + } + @Override public ArgumentEntryType getType() { return ArgumentEntryType.TS_ROLLING; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 3f9b88b8d6..9f5b1cfea1 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -211,10 +211,11 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer DonAsynchron.withCallback(deleteFuture, result -> { calculatedFieldQueueService.pushRequestToQueue(request, request.getKeys(), getCalculatedFieldCallback(request.getCallback(), request.getKeys())); }, safeCallback(getCalculatedFieldCallback(request.getCallback(), request.getKeys())), tsCallBackExecutor); -// addMainCallback(deleteFuture, __ -> request.getCallback().onSuccess(request.getKeys()), request.getCallback()::onFailure); } else { ListenableFuture> deleteFuture = tsService.removeAllLatest(request.getTenantId(), request.getEntityId()); - addMainCallback(deleteFuture, request.getCallback()::onSuccess, request.getCallback()::onFailure); + DonAsynchron.withCallback(deleteFuture, result -> { + calculatedFieldQueueService.pushRequestToQueue(request, request.getKeys(), getCalculatedFieldCallback(request.getCallback(), result)); + }, safeCallback(getCalculatedFieldCallback(request.getCallback(), request.getKeys())), tsCallBackExecutor); } } From 72f26d9fe2b22bbd14fe94ca702c0bc462d2d26f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 26 Feb 2025 15:22:42 +0200 Subject: [PATCH 14/25] fixed functions --- .../cf/ctx/state/TsRollingArgumentEntry.java | 4 +- .../script/api/tbel/TbelCfTsRollingArg.java | 132 +++++++++++++----- 2 files changed, 100 insertions(+), 36 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 90068d6b6f..bb559a3eec 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -25,7 +25,6 @@ import org.thingsboard.script.api.tbel.TbelCfTsDoubleVal; import org.thingsboard.script.api.tbel.TbelCfTsRollingArg; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.exception.CalculatedFieldStateException; import java.util.ArrayList; import java.util.List; @@ -116,10 +115,11 @@ public class TsRollingArgumentEntry implements ArgumentEntry { case STRING -> value.getStrValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); case JSON -> value.getJsonValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); } - cleanupExpiredRecords(); } catch (Exception e) { tsRecords.put(ts, Double.NaN); log.debug("Invalid value '{}' for time series rolling arguments. Only numeric values are supported.", value.getValue()); + } finally { + cleanupExpiredRecords(); } } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java index ceae4e60c5..1bae5399cc 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java @@ -61,14 +61,18 @@ public class TbelCfTsRollingArg implements TbelCfArg, Iterable val) { @@ -97,21 +105,28 @@ public class TbelCfTsRollingArg implements TbelCfArg, Iterable sortedValues = new ArrayList<>(); for (TbelCfTsDoubleVal value : values) { double val = value.getValue(); if (Double.isNaN(val)) { - return Double.NaN; + if (!ignoreNaN) { + return Double.NaN; + } + } else { + sortedValues.add(val); } - sortedValues.add(val); } Collections.sort(sortedValues); @@ -147,51 +172,90 @@ public class TbelCfTsRollingArg implements TbelCfArg, Iterable= 0; i--) { + double prevValue = values.get(i).getValue(); + if (!Double.isNaN(prevValue)) { + return prevValue; + } + } + throw new IllegalArgumentException("Rolling argument values are empty."); } public double first() { + return first(true); + } + + public double first(boolean ignoreNaN) { if (values.isEmpty()) { - return 0; + throw new IllegalArgumentException("Rolling argument values are empty."); } - return hasNaN() ? Double.NaN : values.get(0).getValue(); + double firstValue = values.get(0).getValue(); + if (!Double.isNaN(firstValue) || !ignoreNaN) { + return firstValue; + } + for (int i = 1; i < values.size(); i++) { + double nextValue = values.get(i).getValue(); + if (!Double.isNaN(nextValue)) { + return nextValue; + } + } + throw new IllegalArgumentException("Rolling argument values are empty."); } public double sum() { + return sum(true); + } + + public double sum(boolean ignoreNaN) { if (values.isEmpty()) { - return 0; + throw new IllegalArgumentException("Rolling argument values are empty."); } double sum = 0; for (TbelCfTsDoubleVal value : values) { double val = value.getValue(); if (Double.isNaN(val)) { - return Double.NaN; + if (!ignoreNaN) { + return Double.NaN; + } + } else { + sum += val; } - sum += val; } return sum; } - private boolean hasNaN() { - for (TbelCfTsDoubleVal value : values) { - if (Double.isNaN(value.getValue())) { - return true; - } - } - return false; - } - @JsonIgnore public int getSize() { return values.size(); From 08033fb272006879ca6c9097adb7f21c821d5c31 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Mon, 24 Feb 2025 18:06:32 +0200 Subject: [PATCH 15/25] Calculated fields adjustments arguments table fixes Resolved review comments Resolved review comments minor refactoring refactoring minor refactoring Calculated fields arguments table new UI --- ...lated-field-arguments-table.component.html | 208 +++++++++--------- ...lated-field-arguments-table.component.scss | 54 +++-- ...culated-field-arguments-table.component.ts | 125 ++++++++--- .../calculated-field-dialog.component.ts | 4 +- .../shared/models/calculated-field.models.ts | 5 +- 5 files changed, 226 insertions(+), 170 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index d49f612ae6..ef6a08151a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -16,108 +16,112 @@ -->
-
-
-
{{ 'calculated-fields.argument-name' | translate }}
-
{{ 'calculated-fields.datasource' | translate }}
-
{{ 'common.type' | translate }}
-
{{ 'entity.key' | translate }}
-
-
-
- @for (group of argumentsFormArray.controls; track group) { -
- - - -
- @if (group.get('refEntityId')?.get('id')?.value) { - - - - - {{ entityTypeTranslations.get(group.get('refEntityId').get('entityType').value)?.type | translate }} - - - - - - } @else { - - - - {{ - (group.get('refEntityId')?.get('entityType')?.value === ArgumentEntityType.Tenant - ? 'calculated-fields.argument-current-tenant' - : 'calculated-fields.argument-current') | translate - }} - - - - } -
- - - @if (group.get('refEntityKey').get('type').value; as type) { - - - {{ ArgumentTypeTranslations.get(type) | translate }} - - - } - - - -
- {{ group.get('refEntityKey').get('key').value }} -
- - - -
- - -
-
- } @empty { - {{ 'calculated-fields.no-arguments' | translate }} +
+ + + +
{{ 'common.name' | translate }}
+ + +
{{ argument.argumentName }}
+ + + + + {{ 'entity.entity-type' | translate }} + + +
+ @if (argument.refEntityId?.entityType === ArgumentEntityType.Tenant) { + {{ 'calculated-fields.argument-current-tenant' | translate }} + } @else if (argument.refEntityId?.id) { + {{ entityTypeTranslations.get(argument.refEntityId.entityType).type | translate }} + } @else { + {{ 'calculated-fields.argument-current' | translate }} } -
- @if (errorText) { - - } - -
- +
+ + + + + {{ 'entity-view.target-entity' | translate }} + + +
+ @if (argument.refEntityId?.id) { + + {{ entityNameMap.get(argument.refEntityId.id) ?? '' }} + + } +
+ + + + + {{ 'common.type' | translate }} + + +
{{ ArgumentTypeTranslations.get(argument.refEntityKey.type) | translate }}
+ + + + + {{ 'entity.key' | translate }} + + + +
{{ argument.refEntityKey.key }}
+ + + + + + +
+ + +
+ + + + +
+
+ {{ 'calculated-fields.no-arguments' | translate }}
+ @if (errorText) { + + } +
+
+ +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index febe5233d3..b3d361970a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -13,50 +13,46 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -@use '../../../../../../../scss/constants' as constants; - :host { - .entity-key-field { - border-bottom: 1px solid rgba(0, 0, 0, 0.12); - } + .arguments-table { + min-height: 108px; - .tb-form-table-row-cell-buttons { - --mat-badge-legacy-small-size-container-size: 8px; - --mat-badge-small-size-container-overlap-offset: -5px; - --mat-badge-small-size-text-size: 0; - } + &-with-error { + min-height: 150px; + } - .argument-name-field { - @media #{constants.$mat-sm} { - min-width: 25%; - max-width: 25%; + .mat-mdc-table { + table-layout: fixed; } - @media #{constants.$mat-xs} { - min-width: 33%; - max-width: 33%; + + .key-text { + font-size: 13px; } } - .datasource-field { - min-width: 33%; - max-width: 33%; + .tb-form-table-row-cell-buttons { + --mat-badge-legacy-small-size-container-size: 8px; + --mat-badge-small-size-container-overlap-offset: -5px; + --mat-badge-small-size-text-size: 0; } } :host ::ng-deep { - .entity-field { - a { - font-size: 14px; - white-space: nowrap; - display: block; + .mat-mdc-standard-chip { + .mdc-evolution-chip__cell--primary, .mdc-evolution-chip__action--primary, .mdc-evolution-chip__text-label { overflow: hidden; - text-overflow: ellipsis; } } - .mat-mdc-standard-chip { - .mdc-evolution-chip__cell--primary, .mdc-evolution-chip__action--primary, .mdc-evolution-chip__text-label { - overflow: hidden; + .arguments-table:not(.arguments-table-with-error) { + .mdc-data-table__row:last-child .mat-mdc-cell { + border-bottom: none; + } + } + + .arguments-table { + .mat-mdc-header-row.mat-row-select .mat-mdc-header-cell.entity-type-header { + padding: 0 28px 0 0; } } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index c82a98eaac..5e2afe8f71 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -15,20 +15,22 @@ /// import { + AfterViewInit, ChangeDetectorRef, Component, + DestroyRef, forwardRef, Input, OnChanges, Renderer2, SimpleChanges, + ViewChild, ViewContainerRef, } from '@angular/core'; import { AbstractControl, ControlValueAccessor, FormBuilder, - FormGroup, NG_VALIDATORS, NG_VALUE_ACCESSOR, ValidationErrors, @@ -48,8 +50,11 @@ import { TbPopoverService } from '@shared/components/popover.service'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; -import { isDefined, isDefinedAndNotNull } from '@core/utils'; +import { getEntityDetailsPageURL, isDefined, isDefinedAndNotNull } from '@core/utils'; import { TbPopoverComponent } from '@shared/components/popover.component'; +import { TbTableDatasource } from '@shared/components/table/table-datasource.abstract'; +import { EntityService } from '@core/http/entity.service'; +import { MatSort } from '@angular/material/sort'; @Component({ selector: 'tb-calculated-field-arguments-table', @@ -68,19 +73,23 @@ import { TbPopoverComponent } from '@shared/components/popover.component'; } ], }) -export class CalculatedFieldArgumentsTableComponent implements ControlValueAccessor, Validator, OnChanges { +export class CalculatedFieldArgumentsTableComponent implements ControlValueAccessor, Validator, OnChanges, AfterViewInit { @Input() entityId: EntityId; @Input() tenantId: string; @Input() entityName: string; @Input() calculatedFieldType: CalculatedFieldType; + @ViewChild(MatSort, { static: true }) sort: MatSort; + errorText = ''; argumentsFormArray = this.fb.array([]); + entityNameMap = new Map(); + sortOrder = { direction: 'asc', property: '' }; + dataSource = new CalculatedFieldArgumentDatasource(); readonly entityTypeTranslations = entityTypeTranslations; readonly ArgumentTypeTranslations = ArgumentTypeTranslations; - readonly EntityType = EntityType; readonly ArgumentEntityType = ArgumentEntityType; readonly ArgumentType = ArgumentType; readonly CalculatedFieldType = CalculatedFieldType; @@ -93,10 +102,14 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private popoverService: TbPopoverService, private viewContainerRef: ViewContainerRef, private cd: ChangeDetectorRef, - private renderer: Renderer2 + private renderer: Renderer2, + private entityService: EntityService, + private destroyRef: DestroyRef, ) { - this.argumentsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(() => { - this.propagateChange(this.getArgumentsObject()); + this.argumentsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(value => { + this.updateEntityNameMap(value); + this.updateDataSource(value); + this.propagateChange(this.getArgumentsObject(value)); }); } @@ -107,6 +120,14 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces } } + ngAfterViewInit(): void { + this.sort.sortChange.asObservable().pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { + this.sortOrder.property = this.sort.active; + this.sortOrder.direction = this.sort.direction; + this.updateDataSource(this.argumentsFormArray.value); + }); + } + registerOnChange(fn: (argumentsObj: Record) => void): void { this.propagateChange = fn; } @@ -118,12 +139,13 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces return this.errorText ? { argumentsFormArray: false } : null; } - onDelete(index: number): void { + onDelete($event: Event, index: number): void { + $event.stopPropagation(); this.argumentsFormArray.removeAt(index); this.argumentsFormArray.markAsDirty(); } - manageArgument($event: Event, matButton: MatButton, index?: number): void { + manageArgument($event: Event, matButton: MatButton, argument = {} as CalculatedFieldArgumentValue, index?: number): void { $event?.stopPropagation(); if (this.popoverComponent && !this.popoverComponent.tbHidden) { this.popoverComponent.hide(); @@ -132,16 +154,15 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces if (this.popoverService.hasPopover(trigger)) { this.popoverService.hidePopover(trigger); } else { - const argumentObj = this.argumentsFormArray.at(index)?.getRawValue() ?? {}; const ctx = { index, - argument: argumentObj, + argument, entityId: this.entityId, calculatedFieldType: this.calculatedFieldType, buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', tenantId: this.tenantId, entityName: this.entityName, - usedArgumentNames: this.argumentsFormArray.getRawValue().map(({ argumentName }) => argumentName).filter(name => name !== argumentObj.argumentName), + usedArgumentNames: this.argumentsFormArray.value.map(({ argumentName }) => argumentName).filter(name => name !== argument.argumentName), }; this.popoverComponent = this.popoverService.displayPopover(trigger, this.renderer, this.viewContainerRef, CalculatedFieldArgumentPanelComponent, isDefined(index) ? 'left' : 'right', false, null, @@ -150,7 +171,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces {}, {}, true); this.popoverComponent.tbComponentRef.instance.argumentsDataApplied.subscribe(({ value, index }) => { this.popoverComponent.hide(); - const formGroup = this.getArgumentFormGroup(value); + const formGroup = this.fb.group(value); if (isDefinedAndNotNull(index)) { this.argumentsFormArray.setControl(index, formGroup); } else { @@ -162,9 +183,14 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces } } + private updateDataSource(value: CalculatedFieldArgumentValue[]): void { + const sortedValue = this.sortData(value); + this.dataSource.loadData(sortedValue); + } + private updateErrorText(): void { if (this.calculatedFieldType === CalculatedFieldType.SIMPLE - && this.argumentsFormArray.controls.some(control => control.get('refEntityKey').get('type').value === ArgumentType.Rolling)) { + && this.argumentsFormArray.controls.some(control => control.value.refEntityKey.type === ArgumentType.Rolling)) { this.errorText = 'calculated-fields.hint.arguments-simple-with-rolling'; } else if (!this.argumentsFormArray.controls.length) { this.errorText = 'calculated-fields.hint.arguments-empty'; @@ -173,9 +199,9 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces } } - private getArgumentsObject(): Record { - return this.argumentsFormArray.getRawValue().reduce((acc, rawValue) => { - const { argumentName, ...argument } = rawValue as CalculatedFieldArgumentValue; + private getArgumentsObject(value: CalculatedFieldArgumentValue[]): Record { + return value.reduce((acc, argumentValue) => { + const { argumentName, ...argument } = argumentValue as CalculatedFieldArgumentValue; acc[argumentName] = argument; return acc; }, {} as Record); @@ -186,31 +212,62 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces this.populateArgumentsFormArray(argumentsObj) } + getEntityDetailsPageURL(id: string, type: EntityType): string { + return getEntityDetailsPageURL(id, type); + } + private populateArgumentsFormArray(argumentsObj: Record): void { Object.keys(argumentsObj).forEach(key => { const value: CalculatedFieldArgumentValue = { ...argumentsObj[key], argumentName: key }; - this.argumentsFormArray.push(this.getArgumentFormGroup(value), {emitEvent: false}); + this.argumentsFormArray.push(this.fb.group(value), { emitEvent: false }); + }); + this.argumentsFormArray.updateValueAndValidity(); + } + + private updateEntityNameMap(value: CalculatedFieldArgumentValue[]): void { + value.forEach(({ refEntityId = {}}) => { + if (refEntityId.id && !this.entityNameMap.has(refEntityId.id)) { + const { id, entityType } = refEntityId as EntityId; + this.entityService.getEntity(entityType as EntityType, id, { ignoreLoading: true, ignoreErrors: true }) + .pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe(entity => this.entityNameMap.set(id, entity.name)); + } + }); + } + + private getSortValue(argument: CalculatedFieldArgumentValue, column: string): string { + switch (column) { + case 'entityType': + if (argument.refEntityId?.entityType === ArgumentEntityType.Tenant) { + return 'calculated-fields.argument-current-tenant'; + } else if (argument.refEntityId?.id) { + return entityTypeTranslations.get((argument.refEntityId)?.entityType as unknown as EntityType).type; + } else { + return 'calculated-fields.argument-current'; + } + case 'type': + return ArgumentTypeTranslations.get(argument.refEntityKey.type); + case 'key': + return argument.refEntityKey.key; + default: + return argument.argumentName; + } + } + + private sortData(data: CalculatedFieldArgumentValue[]): CalculatedFieldArgumentValue[] { + return data.sort((a, b) => { + const valA = this.getSortValue(a, this.sortOrder.property) ?? ''; + const valB = this.getSortValue(b, this.sortOrder.property) ?? ''; + return (this.sortOrder.direction === 'asc' ? 1 : -1) * valA.localeCompare(valB); }); } +} - private getArgumentFormGroup(value: CalculatedFieldArgumentValue): FormGroup { - return this.fb.group({ - ...value, - argumentName: [{ value: value.argumentName, disabled: true }], - ...(value.refEntityId ? { - refEntityId: this.fb.group({ - entityType: [{ value: value.refEntityId.entityType, disabled: true }], - id: [{ value: value.refEntityId.id , disabled: true }], - }), - } : {}), - refEntityKey: this.fb.group({ - ...value.refEntityKey, - type: [{ value: value.refEntityKey.type, disabled: true }], - key: [{ value: value.refEntityKey.key, disabled: true }], - }), - }) +class CalculatedFieldArgumentDatasource extends TbTableDatasource { + constructor() { + super(); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index bffb405534..5999b7743d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -143,10 +143,10 @@ export class CalculatedFieldDialogComponent extends DialogComponent, 'label'>, HasVersion, HasTenantId, ExportableEntity { - debugSettings?: EntityDebugSettings; +export interface CalculatedField extends Omit, 'label'>, HasVersion, HasEntityDebugSettings, HasTenantId, ExportableEntity { configuration: CalculatedFieldConfiguration; type: CalculatedFieldType; entityId: EntityId; From 8c86f7fc12cf686ce61e47063875b60ca4a18c90 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 26 Feb 2025 15:48:12 +0200 Subject: [PATCH 16/25] added tests for rolling methods --- .../api/tbel/TbelCfTsRollingArgTest.java | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java new file mode 100644 index 0000000000..e4ebf8a8c1 --- /dev/null +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java @@ -0,0 +1,131 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.within; + +public class TbelCfTsRollingArgTest { + + private final long ts = System.currentTimeMillis(); + + private TbelCfTsRollingArg rollingArg; + + @BeforeEach + void setUp() { + rollingArg = new TbelCfTsRollingArg( + new TbTimeWindow(ts - 30000, ts - 10, 10), + List.of( + new TbelCfTsDoubleVal(ts - 10, Double.NaN), + new TbelCfTsDoubleVal(ts - 20, 2.0), + new TbelCfTsDoubleVal(ts - 30, 8.0), + new TbelCfTsDoubleVal(ts - 40, Double.NaN), + new TbelCfTsDoubleVal(ts - 50, 3.0), + new TbelCfTsDoubleVal(ts - 60, 9.0), + new TbelCfTsDoubleVal(ts - 70, Double.NaN) + ) + ); + } + + @Test + void testMax() { + assertThat(rollingArg.max()).isEqualTo(9.0); + assertThat(rollingArg.max(false)).isNaN(); + } + + @Test + void testMin() { + assertThat(rollingArg.min()).isEqualTo(2.0); + assertThat(rollingArg.min(false)).isNaN(); + } + + @Test + void testMean() { + assertThat(rollingArg.mean()).isEqualTo(5.5); + assertThat(rollingArg.mean(false)).isNaN(); + } + + @Test + void testStd() { + assertThat(rollingArg.std()).isCloseTo(3.0413812651491097, within(0.001)); + assertThat(rollingArg.std(false)).isNaN(); + } + + @Test + void testMedian() { + assertThat(rollingArg.median()).isEqualTo(5.5); + assertThat(rollingArg.median(false)).isNaN(); + } + + @Test + void testCount() { + assertThat(rollingArg.count()).isEqualTo(4); + assertThat(rollingArg.count(false)).isEqualTo(7); + } + + @Test + void testLast() { + assertThat(rollingArg.last()).isEqualTo(9.0); + assertThat(rollingArg.last(false)).isNaN(); + } + + @Test + void testFirst() { + assertThat(rollingArg.first()).isEqualTo(2.0); + assertThat(rollingArg.first(false)).isNaN(); + } + + @Test + void testFirstAndLastWhenOnlyNaNAndIgnoreNaNIsFalse() { + assertThat(rollingArg.first()).isEqualTo(2.0); + rollingArg = new TbelCfTsRollingArg( + new TbTimeWindow(ts - 30000, ts - 10, 10), + List.of( + new TbelCfTsDoubleVal(ts - 10, Double.NaN), + new TbelCfTsDoubleVal(ts - 40, Double.NaN), + new TbelCfTsDoubleVal(ts - 70, Double.NaN) + ) + ); + assertThatThrownBy(rollingArg::first).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::last).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + } + + @Test + void testSum() { + assertThat(rollingArg.sum()).isEqualTo(22.0); + assertThat(rollingArg.sum(false)).isNaN(); + } + + @Test + void testEmptyValues() { + rollingArg = new TbelCfTsRollingArg(new TbTimeWindow(0, 10, 10), List.of()); + assertThatThrownBy(rollingArg::sum).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::max).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::min).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::mean).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::std).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::median).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::first).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::last).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + } + +} \ No newline at end of file From e5555c9d60ba86555c3f02339191ea9d3bfbf564 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 26 Feb 2025 16:11:25 +0200 Subject: [PATCH 17/25] minor fixes --- .../calculatedField/CalculatedFieldEntityMessageProcessor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 1a0f2ddd9a..f7fc204c0f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -197,8 +197,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM processTelemetry(ctx, proto, cfIdList, callback); } else if (proto.getAttrDataCount() > 0) { processAttributes(ctx, proto, cfIdList, callback); - } - if (proto.getRemovedTsKeysCount() > 0) { + } else if (proto.getRemovedTsKeysCount() > 0) { processRemovedTelemetry(ctx, proto, cfIdList, callback); } else if (proto.getRemovedAttrKeysCount() > 0) { processRemovedAttributes(ctx, proto, cfIdList, callback); From 0b44abe121850d2b8eb3cdb9b28bc5bf730c3f69 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Wed, 26 Feb 2025 18:33:26 +0200 Subject: [PATCH 18/25] JS func update --- .../api/tbel/TbelCfTsRollingArgTest.java | 2 +- .../shared/components/js-func.component.ts | 101 ++++++++++++------ 2 files changed, 67 insertions(+), 36 deletions(-) diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java index e4ebf8a8c1..327b477b6f 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java @@ -1,5 +1,5 @@ /** - * Copyright © 2016-2024 The Thingsboard Authors + * Copyright © 2016-2025 The Thingsboard Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ui-ngx/src/app/shared/components/js-func.component.ts b/ui-ngx/src/app/shared/components/js-func.component.ts index 4ea4abea3c..dfb76b4620 100644 --- a/ui-ngx/src/app/shared/components/js-func.component.ts +++ b/ui-ngx/src/app/shared/components/js-func.component.ts @@ -37,7 +37,7 @@ import { ActionNotificationHide, ActionNotificationShow } from '@core/notificati import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { UtilsService } from '@core/services/utils.service'; -import { deepClone, guid, isUndefined, isUndefinedOrNull } from '@app/core/utils'; +import { deepClone, guid, isEqual, isObject, isUndefined, isUndefinedOrNull } from '@app/core/utils'; import { TranslateService } from '@ngx-translate/core'; import { CancelAnimationFrame, RafService } from '@core/services/raf.service'; import { TbEditorCompleter } from '@shared/models/ace/completion.models'; @@ -179,26 +179,12 @@ export class JsFuncComponent implements OnInit, OnChanges, OnDestroy, ControlVal private http: HttpClient) { } - ngOnChanges(changes: SimpleChanges): void { - if (changes.functionArgs) { - this.updateFunctionArgsString(); - this.updateFunctionLabel(); - } - if (changes.editorCompleter?.previousValue) { - this.updateCompleters(); - } - if (changes.highlightRules?.previousValue) { - this.updateHighlightRules(); - } - } - ngOnInit(): void { - if (this.functionTitle || this.label) { - this.hideBrackets = true; - } if (!this.resultType || this.resultType.length === 0) { this.resultType = 'nocheck'; } + this.updateFunctionArgsString() + this.updateFunctionLabel(); const editorElement = this.javascriptEditorElmRef.nativeElement; let editorOptions: Partial = { mode: 'ace/mode/javascript', @@ -262,6 +248,16 @@ export class JsFuncComponent implements OnInit, OnChanges, OnDestroy, ControlVal ); } + ngOnChanges(changes: SimpleChanges): void { + for (const propName of Object.keys(changes)) { + const { firstChange, currentValue, previousValue } = changes[propName]; + const isChanged = isObject(currentValue) ? isEqual(currentValue, previousValue) : currentValue !== previousValue; + if (!firstChange && isChanged) { + this.updateByChangesPropName(propName); + } + } + } + ngOnDestroy(): void { if (this.editorResize$) { this.editorResize$.disconnect(); @@ -271,24 +267,6 @@ export class JsFuncComponent implements OnInit, OnChanges, OnDestroy, ControlVal } } - private updateHighlightRules(): void { - // @ts-ignore - if (!!this.highlightRules && !!this.jsEditor.session.$mode) { - // @ts-ignore - const newMode = new this.jsEditor.session.$mode.constructor(); - newMode.$highlightRules = new newMode.HighlightRules(); - for(const group in this.highlightRules) { - if(!!newMode.$highlightRules.$rules[group]) { - newMode.$highlightRules.$rules[group].unshift(...this.highlightRules[group]); - } else { - newMode.$highlightRules.$rules[group] = this.highlightRules[group]; - } - } - // @ts-ignore - this.jsEditor.session.$onChangeMode(newMode); - } - } - private onAceEditorResize() { if (this.editorsResizeCaf) { this.editorsResizeCaf(); @@ -340,6 +318,9 @@ export class JsFuncComponent implements OnInit, OnChanges, OnDestroy, ControlVal } private updateFunctionLabel(): void { + if (this.functionTitle || this.label) { + this.hideBrackets = true; + } if (this.functionTitle) { this.functionLabel = `${this.functionTitle}: f(${this.functionArgsString})`; } else if (this.label) { @@ -351,6 +332,10 @@ export class JsFuncComponent implements OnInit, OnChanges, OnDestroy, ControlVal this.cd.markForCheck(); } + private updatedScriptLanguage() { + this.jsEditor.session.setMode(`ace/mode/${ScriptLanguage.TBEL === this.scriptLanguage ? 'tbel' : 'javascript'}`); + } + validateOnSubmit(): Observable { if (!this.disabled) { this.cleanupJsErrors(); @@ -561,6 +546,52 @@ export class JsFuncComponent implements OnInit, OnChanges, OnDestroy, ControlVal } } + private updateByChangesPropName(propName: string): void { + switch (propName) { + case 'functionArgs': + this.updateFunctionArgsString() + this.updateFunctionLabel(); + this.updateJsWorkerGlobals(); + break; + case 'label': + case 'functionTitle': + case 'functionName': + this.updateFunctionLabel(); + break; + case 'scriptLanguage': + this.updatedScriptLanguage(); + break; + case 'disableUndefinedCheck': + case 'globalVariables': + this.updateJsWorkerGlobals(); + break; + case 'editorCompleter': + this.updateCompleters(); + break; + case 'highlightRules': + this.updateHighlightRules(); + break; + } + } + + private updateHighlightRules(): void { + // @ts-ignore + if (!!this.highlightRules && !!this.jsEditor.session.$mode) { + // @ts-ignore + const newMode = new this.jsEditor.session.$mode.constructor(); + newMode.$highlightRules = new newMode.HighlightRules(); + for(const group in this.highlightRules) { + if(!!newMode.$highlightRules.$rules[group]) { + newMode.$highlightRules.$rules[group].unshift(...this.highlightRules[group]); + } else { + newMode.$highlightRules.$rules[group] = this.highlightRules[group]; + } + } + // @ts-ignore + this.jsEditor.session.$onChangeMode(newMode); + } + } + private updateJsWorkerGlobals() { // @ts-ignore if (!!this.jsEditor.session.$worker) { From fb2c396917da35e0edcc4be5b07aba788afaf764 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Wed, 26 Feb 2025 18:35:33 +0200 Subject: [PATCH 19/25] minor refactoring --- ui-ngx/src/app/shared/components/js-func.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/shared/components/js-func.component.ts b/ui-ngx/src/app/shared/components/js-func.component.ts index dfb76b4620..7c7d14eff5 100644 --- a/ui-ngx/src/app/shared/components/js-func.component.ts +++ b/ui-ngx/src/app/shared/components/js-func.component.ts @@ -251,7 +251,7 @@ export class JsFuncComponent implements OnInit, OnChanges, OnDestroy, ControlVal ngOnChanges(changes: SimpleChanges): void { for (const propName of Object.keys(changes)) { const { firstChange, currentValue, previousValue } = changes[propName]; - const isChanged = isObject(currentValue) ? isEqual(currentValue, previousValue) : currentValue !== previousValue; + const isChanged = isObject(currentValue) ? !isEqual(currentValue, previousValue) : currentValue !== previousValue; if (!firstChange && isChanged) { this.updateByChangesPropName(propName); } From b0604ff9c72b50f7cb519c73c6e24875738082dd Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 27 Feb 2025 11:23:23 +0200 Subject: [PATCH 20/25] removed resourse CALCULATED_FIELD --- .../server/service/security/permission/Resource.java | 4 +--- .../service/security/permission/TenantAdminPermissions.java | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java index 8ed30b7b61..4cb281a719 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java @@ -50,9 +50,7 @@ public enum Resource { VERSION_CONTROL, NOTIFICATION(EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE), - MOBILE_APP_SETTINGS, - CALCULATED_FIELD(EntityType.CALCULATED_FIELD), - CALCULATED_FIELD_LINK(EntityType.CALCULATED_FIELD_LINK); + MOBILE_APP_SETTINGS; private final Set entityTypes; diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index a072cf2738..7a67d6739e 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -55,7 +55,6 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.OAUTH2_CONFIGURATION_TEMPLATE, new PermissionChecker.GenericPermissionChecker(Operation.READ)); put(Resource.MOBILE_APP, tenantEntityPermissionChecker); put(Resource.MOBILE_APP_BUNDLE, tenantEntityPermissionChecker); - put(Resource.CALCULATED_FIELD, tenantEntityPermissionChecker); } public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { From 729d8c0a78823fbf90cb3ea023c67e999e90155d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 27 Feb 2025 13:41:10 +0200 Subject: [PATCH 21/25] added resource --- .../server/service/security/permission/Resource.java | 3 ++- .../service/security/permission/TenantAdminPermissions.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java index 4cb281a719..9d7590f786 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java @@ -50,7 +50,8 @@ public enum Resource { VERSION_CONTROL, NOTIFICATION(EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE), - MOBILE_APP_SETTINGS; + MOBILE_APP_SETTINGS, + CALCULATED_FIELD(EntityType.CALCULATED_FIELD); private final Set entityTypes; diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 7a67d6739e..a072cf2738 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -55,6 +55,7 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.OAUTH2_CONFIGURATION_TEMPLATE, new PermissionChecker.GenericPermissionChecker(Operation.READ)); put(Resource.MOBILE_APP, tenantEntityPermissionChecker); put(Resource.MOBILE_APP_BUNDLE, tenantEntityPermissionChecker); + put(Resource.CALCULATED_FIELD, tenantEntityPermissionChecker); } public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { From 62da129d6bb08e67243084af811f4dacee083bf6 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 27 Feb 2025 16:46:55 +0200 Subject: [PATCH 22/25] Review comments fixed --- ...alculatedFieldManagerMessageProcessor.java | 14 ++-- .../EntityCalculatedFieldTelemetryMsg.java | 1 + .../controller/CalculatedFieldController.java | 5 +- .../service/cf/CalculatedFieldCache.java | 1 - ...faultCalculatedFieldProcessingService.java | 6 -- .../DefaultCalculatedFieldQueueService.java | 18 +---- ...aultCalculatedFieldEntityProfileCache.java | 2 +- .../service/cf/ctx/state/ArgumentEntry.java | 12 +-- .../KafkaCalculatedFieldStateService.java | 5 +- .../RocksDBCalculatedFieldStateService.java | 7 +- .../ctx/state/SingleValueArgumentEntry.java | 3 - ...CalculatedFieldAttributeUpdateRequest.java | 72 ------------------ ...CalculatedFieldTelemetryUpdateRequest.java | 39 ---------- ...alculatedFieldTimeSeriesUpdateRequest.java | 75 ------------------- .../entitiy/AbstractTbEntityService.java | 10 --- ...faultTbCalculatedFieldConsumerService.java | 13 +--- .../queue/DefaultTbClusterService.java | 4 +- .../server/utils/CalculatedFieldUtils.java | 8 ++ .../server/actors/TbActorMailbox.java | 5 +- .../server/dao/cf/CalculatedFieldService.java | 15 ---- .../common/data/cf/CalculatedFieldLink.java | 6 +- .../cf/CalculatedFieldLinkConfiguration.java | 31 -------- .../BaseCalculatedFieldConfiguration.java | 31 -------- .../CalculatedFieldConfiguration.java | 4 - .../common/data/id/CalculatedFieldLinkId.java | 2 +- .../thingsboard/server/common/msg/TbMsg.java | 2 +- common/proto/src/main/proto/queue.proto | 2 +- .../DefaultTbServiceInfoProvider.java | 7 +- .../provider/TbQueueProducerProvider.java | 2 +- .../DefaultTbQueueRequestTemplateTest.java | 18 +---- .../discovery/ZkDiscoveryServiceTest.java | 6 +- .../script/api/tbel/TbelCfSingleValueArg.java | 8 +- .../dao/cf/BaseCalculatedFieldService.java | 46 ------------ .../server/dao/cf/CalculatedFieldDao.java | 2 - .../server/dao/cf/CalculatedFieldLinkDao.java | 3 - .../server/dao/model/ModelConstants.java | 1 - .../model/sql/CalculatedFieldLinkEntity.java | 20 ----- ...efaultNativeCalculatedFieldRepository.java | 2 - .../dao/sql/cf/JpaCalculatedFieldDao.java | 5 -- .../dao/sql/cf/JpaCalculatedFieldLinkDao.java | 5 -- .../main/resources/sql/schema-entities.sql | 1 - pom.xml | 2 +- 42 files changed, 57 insertions(+), 464 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 3395f279a1..d26d89626e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -65,8 +65,8 @@ import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; public class CalculatedFieldManagerMessageProcessor extends AbstractContextAwareMsgProcessor { private final Map calculatedFields = new HashMap<>(); - private final Map> entityIdCalculatedFields = new ConcurrentHashMap<>(); - private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); + private final Map> entityIdCalculatedFields = new HashMap<>(); + private final Map> entityIdCalculatedFieldLinks = new HashMap<>(); private final CalculatedFieldProcessingService cfExecService; private final CalculatedFieldStateService cfStateService; @@ -105,7 +105,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware calculatedFields.put(cf.getId(), cfCtx); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) - entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new ArrayList<>()).add(cfCtx); + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); msg.getCallback().onSuccess(); } @@ -114,7 +114,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var link = msg.getLink(); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) - entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).add(link); + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link); msg.getCallback().onSuccess(); } @@ -262,7 +262,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } calculatedFields.put(newCf.getId(), newCfCtx); List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); - List newCfList = new ArrayList<>(oldCfList.size()); + List newCfList = new CopyOnWriteArrayList<>(); boolean found = false; for (CalculatedFieldCtx oldCtx : oldCfList) { if (oldCtx.getCfId().equals(newCf.getId())) { @@ -456,13 +456,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void addLinks(CalculatedField newCf) { var newLinks = newCf.getConfiguration().buildCalculatedFieldLinks(tenantId, newCf.getEntityId(), newCf.getId()); - newLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).add(link)); + newLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link)); } private void deleteLinks(CalculatedFieldCtx cfCtx) { var oldCf = cfCtx.getCalculatedField(); var oldLinks = oldCf.getConfiguration().buildCalculatedFieldLinks(tenantId, oldCf.getEntityId(), oldCf.getId()); - oldLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).remove(link)); + oldLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).remove(link)); } public void onPartitionChange(CalculatedFieldPartitionChangeMsg msg) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java index 8aab7d2743..8ded4b6028 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java @@ -32,6 +32,7 @@ public class EntityCalculatedFieldTelemetryMsg implements ToCalculatedFieldSyste private final TenantId tenantId; private final EntityId entityId; private final CalculatedFieldTelemetryMsgProto proto; + // Both lists are effectively immutable in CalculatedFieldManagerMessageProcessor and must stay so. private final List entityIdFields; private final List profileIdFields; private final TbCallback callback; diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index b1e9dc9214..fbeade5819 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -35,6 +35,7 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.cf.CalculatedField; @@ -53,6 +54,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldTbelScriptEngi import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; import java.util.ArrayList; import java.util.Collections; @@ -127,6 +129,7 @@ public class CalculatedFieldController extends BaseController { public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") @RequestBody CalculatedField calculatedField) throws Exception { calculatedField.setTenantId(getTenantId()); + checkEntity(calculatedField.getId(), calculatedField, Resource.CALCULATED_FIELD); checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); checkReferencedEntities(calculatedField.getConfiguration(), getCurrentUser()); return tbCalculatedFieldService.save(calculatedField, getCurrentUser()); @@ -176,7 +179,7 @@ public class CalculatedFieldController extends BaseController { public void deleteCalculatedField(@PathVariable(CALCULATED_FIELD_ID) String strCalculatedField) throws Exception { checkParameter(CALCULATED_FIELD_ID, strCalculatedField); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedField)); - CalculatedField calculatedField = tbCalculatedFieldService.findById(calculatedFieldId, getCurrentUser()); + CalculatedField calculatedField = checkCalculatedFieldId(calculatedFieldId, Operation.DELETE); checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); tbCalculatedFieldService.delete(calculatedField, getCurrentUser()); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index 3fbfe26dc0..fb63432fed 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -23,7 +23,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.List; -import java.util.Set; public interface CalculatedFieldCache { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index 5ca4993d3f..f69f5cd50b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -21,11 +21,9 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; -import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.math.NumberUtils; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; @@ -104,10 +102,6 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP private ListeningExecutorService calculatedFieldCallbackExecutor; - @Value("${calculatedField.initFetchPackSize:50000}") - @Getter - private int initFetchPackSize; - @PostConstruct public void init() { calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 7421dfe187..1b52032bb5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -16,10 +16,8 @@ package org.thingsboard.server.service.cf; import com.google.common.util.concurrent.FutureCallback; -import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.rule.engine.api.AttributesDeleteRequest; import org.thingsboard.rule.engine.api.AttributesSaveRequest; @@ -40,7 +38,6 @@ import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; @@ -59,6 +56,7 @@ import java.util.function.Predicate; import java.util.function.Supplier; import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; +import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; @Service @Slf4j @@ -84,17 +82,10 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT ); - @Value("${calculatedField.initFetchPackSize:50000}") - @Getter - private int initFetchPackSize; - @Override public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); - //TODO: 1. check that request entity has calculated fields for entity or profile. If yes - push to corresponding partitions; - //TODO: 2. check that request entity has calculated field links. If yes - push to corresponding partitions; - //TODO: in 1 and 2 we should do the check as quick as possible. Should we also check the field/link keys?; checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -239,13 +230,6 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS return telemetryMsg; } - private CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { - return CalculatedFieldIdProto.newBuilder() - .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) - .build(); - } - private static TbQueueCallback wrap(FutureCallback callback) { if (callback != null) { return new FutureCallbackWrapper(callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java index 4377ee58aa..4cf62c01b3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java @@ -38,7 +38,7 @@ import java.util.stream.Collectors; @Service @Slf4j @RequiredArgsConstructor -//TODO: remove and use TenantEntityProfileCache in each CalculatedFieldManagerMessageProcessor; +//TODO ashvayka: remove and use TenantEntityProfileCache in each CalculatedFieldManagerMessageProcessor; public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEventListener implements CalculatedFieldEntityProfileCache { private static final Integer UNKNOWN = 0; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 6f827065af..83e10b8194 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -44,6 +44,12 @@ public interface ArgumentEntry { boolean isEmpty(); + TbelCfArg toTbelCfArg(); + + boolean isForceResetPrevious(); + + void setForceResetPrevious(boolean forceResetPrevious); + static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { return new SingleValueArgumentEntry(kvEntry); } @@ -52,10 +58,4 @@ public interface ArgumentEntry { return new TsRollingArgumentEntry(kvEntries, limit, timeWindow); } - TbelCfArg toTbelCfArg(); - - boolean isForceResetPrevious(); - - void setForceResetPrevious(boolean forceResetPrevious); - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index 0c2b5ac07e..edb47dd7fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -44,10 +44,7 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; -import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.bytesToString; -import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.bytesToUuid; -import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.stringToBytes; -import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.uuidToBytes; +import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.*; @Service @RequiredArgsConstructor diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index 32ad6df551..a508eecada 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -37,7 +37,7 @@ public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldS private final CfRocksDb cfRocksDb; - private Set partitions; + private boolean initialized; @Override protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback) { @@ -53,7 +53,7 @@ public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldS @Override public void restore(Set partitions) { - if (this.partitions == null) { + if (!this.initialized) { cfRocksDb.forEach((key, value) -> { try { processRestoredState(CalculatedFieldStateProto.parseFrom(value)); @@ -61,10 +61,9 @@ public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldS log.error("[{}] Failed to process restored state", key, e); } }); + this.initialized = true; } - eventConsumer.update(partitions); - this.partitions = partitions; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 2e8bb63f5f..a064a99935 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -100,8 +100,6 @@ public class SingleValueArgumentEntry implements ArgumentEntry { if (newVersion == null || this.version == null || newVersion > this.version) { this.ts = singleValueEntry.getTs(); this.version = newVersion; - - // TODO: should we persist updated ts and version values? BasicKvEntry newValue = singleValueEntry.getKvEntryValue(); if (this.kvEntryValue != null && this.kvEntryValue.getValue().equals(newValue.getValue())) { return false; @@ -109,7 +107,6 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.kvEntryValue = singleValueEntry.getKvEntryValue(); return true; } - } else { throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java deleted file mode 100644 index f95b9041a3..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf.telemetry; - -import lombok.AllArgsConstructor; -import lombok.Data; -import org.thingsboard.rule.engine.api.AttributesSaveRequest; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.util.TbPair; -import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Data -@AllArgsConstructor -public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { - - private TenantId tenantId; - private EntityId entityId; - private AttributeScope scope; - private List kvEntries; - private List previousCalculatedFieldIds; - - public CalculatedFieldAttributeUpdateRequest(AttributesSaveRequest request) { - this.tenantId = request.getTenantId(); - this.entityId = request.getEntityId(); - this.scope = request.getScope(); - this.kvEntries = request.getEntries(); - this.previousCalculatedFieldIds = request.getPreviousCalculatedFieldIds(); - } - - @Override - public Map getMappedTelemetry(CalculatedFieldCtx ctx, EntityId referencedEntityId) { - Map mappedKvEntries = new HashMap<>(); - Map, String> referencedKeys = ctx.getReferencedEntityKeys(); - - kvEntries.forEach(entry -> { - String key = entry.getKey(); - - ReferencedEntityKey referencedEntityKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); - - String argName = referencedKeys.get(new TbPair<>(referencedEntityId, referencedEntityKey)); - - if (argName != null) { - mappedKvEntries.put(argName, entry); - } - }); - - return mappedKvEntries; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java deleted file mode 100644 index 9f1043c4d3..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf.telemetry; - -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; - -import java.util.List; -import java.util.Map; - -public interface CalculatedFieldTelemetryUpdateRequest { - - TenantId getTenantId(); - - EntityId getEntityId(); - - List getKvEntries(); - - List getPreviousCalculatedFieldIds(); - - Map getMappedTelemetry(CalculatedFieldCtx ctx, EntityId referencedEntityId); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java deleted file mode 100644 index 3f85108e0c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf.telemetry; - -import lombok.AllArgsConstructor; -import lombok.Data; -import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; -import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.util.TbPair; -import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Data -@AllArgsConstructor -public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { - - private TenantId tenantId; - private EntityId entityId; - private List kvEntries; - private List previousCalculatedFieldIds; - - public CalculatedFieldTimeSeriesUpdateRequest(TimeseriesSaveRequest request) { - this.tenantId = request.getTenantId(); - this.entityId = request.getEntityId(); - this.kvEntries = request.getEntries(); - this.previousCalculatedFieldIds = request.getPreviousCalculatedFieldIds(); - } - - @Override - public Map getMappedTelemetry(CalculatedFieldCtx ctx, EntityId referencedEntityId) { - Map mappedKvEntries = new HashMap<>(); - Map, String> referencedKeys = ctx.getReferencedEntityKeys(); - - kvEntries.forEach(entry -> { - String key = entry.getKey(); - - ReferencedEntityKey tsLatestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); - String argTsLatestName = referencedKeys.get(new TbPair<>(referencedEntityId, tsLatestKey)); - - if (argTsLatestName != null) { - mappedKvEntries.put(argTsLatestName, entry); - } else { - ReferencedEntityKey tsRollingKey = new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null); - String argTsRollingName = referencedKeys.get(new TbPair<>(referencedEntityId, tsRollingKey)); - - if (argTsRollingName != null) { - mappedKvEntries.put(argTsRollingName, entry); - } - } - }); - - return mappedKvEntries; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index 99e62d9186..e9ee7202a7 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -78,16 +78,6 @@ public abstract class AbstractTbEntityService { @Lazy private EntitiesVersionControlService vcService; @Autowired - protected TenantService tenantService; - @Autowired - protected AssetService assetService; - @Autowired - protected DeviceService deviceService; - @Autowired - protected AssetProfileService assetProfileService; - @Autowired - protected DeviceProfileService deviceProfileService; - @Autowired protected EntityService entityService; protected boolean isTestProfile() { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index bc15a64770..e76a2be9be 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -87,8 +87,6 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer private PartitionedQueueConsumerManager> eventConsumer; - private ListeningExecutorService calculatedFieldsExecutor; - public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, ActorSystemContext actorContext, TbDeviceProfileCache deviceProfileCache, @@ -109,7 +107,6 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer @PostConstruct public void init() { super.init("tb-cf"); - this.calculatedFieldsExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(poolSize, "tb-cf-executor")); // TODO: multiple threads. this.eventConsumer = PartitionedQueueConsumerManager.>create() .queueKey(QueueKey.CF) @@ -127,9 +124,6 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer @PreDestroy public void destroy() { super.destroy(); - if (calculatedFieldsExecutor != null) { - calculatedFieldsExecutor.shutdownNow(); - } } @Override @@ -197,10 +191,9 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer packSubmitFuture.cancel(true); log.info("Timeout to process message: {}", pendingMsgHolder.getMsg()); } -// if (log.isDebugEnabled()) { -// ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); -// } - ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); // TODO: replace with commented above after testing + if (log.isDebugEnabled()) { + ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); + } ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } consumer.commit(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 0ccfb764e2..800578ace4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -196,12 +196,12 @@ public class DefaultTbClusterService implements TbClusterService { UUID msgId = UUID.randomUUID(); TbQueueProducer> toCfProducer = producerProvider.getCalculatedFieldsNotificationsMsgProducer(); Set tbReServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); + MultipleTbQueueCallbackWrapper callbackWrapper = new MultipleTbQueueCallbackWrapper(tbReServices.size(), callback); for (String serviceId : tbReServices) { TopicPartitionInfo tpi = topicService.getCalculatedFieldNotificationsTopic(serviceId); - toCfProducer.send(tpi, new TbProtoQueueMsg<>(msgId, toCfMsg), null); + toCfProducer.send(tpi, new TbProtoQueueMsg<>(msgId, toCfMsg), callbackWrapper); toRuleEngineNfs.incrementAndGet(); } - callback.onSuccess(null); // TODO: refactor to be fair, similar to multi-value callback; } @Override diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 6f10c63492..77080c28c8 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.util.KvProtoUtil; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.TsDoubleValProto; @@ -42,6 +43,13 @@ import java.util.UUID; public class CalculatedFieldUtils { + public static CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { + return CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) + .build(); + } + public static CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { return CalculatedFieldEntityCtxIdProto.newBuilder() .setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java index af6031fb83..4b822e0030 100644 --- a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java @@ -153,14 +153,13 @@ public final class TbActorMailbox implements TbActorCtx { } if (msg != null) { try { - log.debug("[{}] Going to process message: {}", selfId, msg); + log.trace("[{}] Going to process message: {}", selfId, msg); actor.process(msg); } catch (TbRuleNodeUpdateException updateException) { stopReason = TbActorStopReason.INIT_FAILED; destroy(updateException.getCause()); } catch (Throwable t) { - //TODO: revert; - log.error("[{}] Failed to process message: {}", selfId, msg, t); + log.debug("[{}] Failed to process message: {}", selfId, msg, t); ProcessFailureStrategy strategy = actor.onProcessFailure(msg, t); if (strategy.isStop()) { system.stop(selfId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index adc794e2c1..3d91790790 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.cf; -import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -34,14 +33,10 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); - ListenableFuture findCalculatedFieldByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); - List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); - List findAllCalculatedFields(); - PageData findAllCalculatedFields(PageLink pageLink); PageData findAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink); @@ -54,22 +49,12 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); - ListenableFuture findCalculatedFieldLinkByIdAsync(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); - - List findAllCalculatedFieldLinks(); - List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId); List findAllCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); - ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); - PageData findAllCalculatedFieldLinks(PageLink pageLink); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); - boolean referencedInAnyCalculatedFieldIncludingEntityId(TenantId tenantId, EntityId referencedEntityId); - - boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId); - } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java index ce979214d2..3f048815da 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java @@ -36,8 +36,6 @@ public class CalculatedFieldLink extends BaseData { @Schema(description = "JSON object with the Calculated Field Id. ", accessMode = Schema.AccessMode.READ_ONLY) private CalculatedFieldId calculatedFieldId; - @Schema - private transient CalculatedFieldLinkConfiguration configuration; public CalculatedFieldLink() { super(); @@ -47,11 +45,10 @@ public class CalculatedFieldLink extends BaseData { super(id); } - public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFieldLinkConfiguration configuration) { + public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId) { this.tenantId = tenantId; this.entityId = entityId; this.calculatedFieldId = calculatedFieldId; - this.configuration = configuration; } @Override @@ -61,7 +58,6 @@ public class CalculatedFieldLink extends BaseData { .append("tenantId=").append(tenantId) .append(", entityId=").append(entityId) .append(", calculatedFieldId=").append(calculatedFieldId) - .append(", configuration=").append(configuration) .append(", createdTime=").append(createdTime) .append(", id=").append(id).append(']') .toString(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java deleted file mode 100644 index b3aaa46b86..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.cf; - -import lombok.Data; - -import java.util.HashMap; -import java.util.Map; - -@Data -public class CalculatedFieldLinkConfiguration { - - private Map clientAttributes = new HashMap<>(); - private Map serverAttributes = new HashMap<>(); - private Map sharedAttributes = new HashMap<>(); - private Map timeSeries = new HashMap<>(); - -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index 8ff4fab94a..8227ff4603 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -17,7 +17,6 @@ package org.thingsboard.server.common.data.cf.configuration; import lombok.Data; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -42,35 +41,6 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel .collect(Collectors.toList()); } - @Override - public CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId) { - CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); - - arguments.entrySet().stream() - .filter(entry -> entry.getValue().getRefEntityId() != null && entry.getValue().getRefEntityId().equals(entityId)) - .forEach(entry -> { - ReferencedEntityKey refEntityKey = entry.getValue().getRefEntityKey(); - String argumentName = entry.getKey(); - - switch (refEntityKey.getType()) { - case ATTRIBUTE -> { - switch (refEntityKey.getScope()) { - case CLIENT_SCOPE -> - linkConfiguration.getClientAttributes().put(refEntityKey.getKey(), argumentName); - case SERVER_SCOPE -> - linkConfiguration.getServerAttributes().put(refEntityKey.getKey(), argumentName); - case SHARED_SCOPE -> - linkConfiguration.getSharedAttributes().put(refEntityKey.getKey(), argumentName); - } - } - case TS_LATEST, TS_ROLLING -> - linkConfiguration.getTimeSeries().put(refEntityKey.getKey(), argumentName); - } - }); - - return linkConfiguration; - } - @Override public List buildCalculatedFieldLinks(TenantId tenantId, EntityId cfEntityId, CalculatedFieldId calculatedFieldId) { return getReferencedEntities().stream() @@ -85,7 +55,6 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel link.setTenantId(tenantId); link.setEntityId(referencedEntityId); link.setCalculatedFieldId(calculatedFieldId); - link.setConfiguration(getReferencedEntityConfig(referencedEntityId)); return link; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index b97024c47f..c53f1fe5f1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -53,9 +52,6 @@ public interface CalculatedFieldConfiguration { @JsonIgnore List getReferencedEntities(); - @JsonIgnore - CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId); - List buildCalculatedFieldLinks(TenantId tenantId, EntityId cfEntityId, CalculatedFieldId calculatedFieldId); CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, EntityId referencedEntityId, CalculatedFieldId calculatedFieldId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java index 75a0dd8b90..6a0c680bb6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.EntityType; import java.util.UUID; @Schema -public class CalculatedFieldLinkId extends UUIDBased implements EntityId { +public class CalculatedFieldLinkId extends UUIDBased implements EntityId { private static final long serialVersionUID = 1L; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index c0ec06191d..f1c2236080 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -485,7 +485,7 @@ public final class TbMsg implements Serializable { } public TbMsgBuilder previousCalculatedFieldIds(List previousCalculatedFieldIds) { - this.previousCalculatedFieldIds = previousCalculatedFieldIds; + this.previousCalculatedFieldIds = new CopyOnWriteArrayList<>(previousCalculatedFieldIds); return this; } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index fe412dbc0c..f41bf92592 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1190,7 +1190,7 @@ message ComponentLifecycleMsgProto { int64 entityIdMSB = 4; int64 entityIdLSB = 5; ComponentLifecycleEvent event = 6; - //Since 4.0. To replace the + //Since 4.0. TODO: replace the DeviceNameOrTypeUpdateMsgProto string oldName = 7; string name = 8; int64 oldProfileIdMSB = 9; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index a2753b8832..4bcbb5152c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -39,12 +39,7 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; -import static org.thingsboard.common.util.SystemUtil.getCpuCount; -import static org.thingsboard.common.util.SystemUtil.getCpuUsage; -import static org.thingsboard.common.util.SystemUtil.getDiscSpaceUsage; -import static org.thingsboard.common.util.SystemUtil.getMemoryUsage; -import static org.thingsboard.common.util.SystemUtil.getTotalDiscSpace; -import static org.thingsboard.common.util.SystemUtil.getTotalMemory; +import static org.thingsboard.common.util.SystemUtil.*; @Component diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index de2e372cbf..865637b2ff 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -15,8 +15,8 @@ */ package org.thingsboard.server.queue.provider; -import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index 1760555e07..d3975c78bd 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -37,23 +37,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.*; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.BDDMockito.willAnswer; -import static org.mockito.BDDMockito.willDoNothing; -import static org.mockito.BDDMockito.willReturn; -import static org.mockito.Mockito.RETURNS_DEEP_STUBS; -import static org.mockito.Mockito.atLeastOnce; -import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; +import static org.mockito.BDDMockito.*; +import static org.mockito.Mockito.*; import static org.mockito.hamcrest.MockitoHamcrest.longThat; @Slf4j diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/discovery/ZkDiscoveryServiceTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/discovery/ZkDiscoveryServiceTest.java index c8d91b0c0f..19643f4ad4 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/discovery/ZkDiscoveryServiceTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/discovery/ZkDiscoveryServiceTest.java @@ -42,11 +42,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; @ExtendWith(MockitoExtension.class) public class ZkDiscoveryServiceTest { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java index 4b84f1a642..193b6ea1ae 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java @@ -22,6 +22,8 @@ import lombok.Data; @Data public class TbelCfSingleValueArg implements TbelCfArg { + public static final long OBJ_SIZE = 32L; // Approximate calculation; + private final long ts; private final Object value; @@ -36,7 +38,11 @@ public class TbelCfSingleValueArg implements TbelCfArg { @Override public long memorySize() { - return 8L; // TODO; + if (value instanceof String strValue) { + return OBJ_SIZE + strValue.length(); + } else { + return OBJ_SIZE; + } } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 9024ae1a70..416bff6ea3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -84,13 +84,6 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldDao.findById(tenantId, calculatedFieldId.getId()); } - @Override - public ListenableFuture findCalculatedFieldByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - log.trace("Executing findCalculatedFieldByIdAsync [{}]", calculatedFieldId); - validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); - return calculatedFieldDao.findByIdAsync(tenantId, calculatedFieldId.getId()); - } - @Override public List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId) { log.trace("Executing findCalculatedFieldIdsByEntityId [{}]", entityId); @@ -105,12 +98,6 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldDao.findCalculatedFieldsByEntityId(tenantId, entityId); } - @Override - public List findAllCalculatedFields() { - log.trace("Executing findAll"); - return calculatedFieldDao.findAll(); - } - @Override public PageData findAllCalculatedFields(PageLink pageLink) { log.trace("Executing findAll, pageLink [{}]", pageLink); @@ -176,20 +163,6 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findById(tenantId, calculatedFieldLinkId.getId()); } - @Override - public ListenableFuture findCalculatedFieldLinkByIdAsync(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId) { - log.trace("Executing findCalculatedFieldLinkByIdAsync [{}]", calculatedFieldLinkId); - validateId(tenantId, id -> INCORRECT_TENANT_ID + id); - validateId(calculatedFieldLinkId, id -> "Incorrect calculatedFieldLinkId " + id); - return calculatedFieldLinkDao.findByIdAsync(tenantId, calculatedFieldLinkId.getId()); - } - - @Override - public List findAllCalculatedFieldLinks() { - log.trace("Executing findAllCalculatedFieldLinks"); - return calculatedFieldLinkDao.findAll(); - } - @Override public List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId) { log.trace("Executing findAllCalculatedFieldLinksById, calculatedFieldId [{}]", calculatedFieldId); @@ -202,12 +175,6 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findCalculatedFieldLinksByEntityId(tenantId, entityId); } - @Override - public ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - log.trace("Executing findAllCalculatedFieldLinksByIdAsync, calculatedFieldId [{}]", calculatedFieldId); - return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldIdAsync(tenantId, calculatedFieldId); - } - @Override public PageData findAllCalculatedFieldLinks(PageLink pageLink) { log.trace("Executing findAllCalculatedFieldLinks, pageLink [{}]", pageLink); @@ -224,19 +191,6 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); } - @Override - public boolean referencedInAnyCalculatedFieldIncludingEntityId(TenantId tenantId, EntityId referencedEntityId) { - return calculatedFieldDao.findAllByTenantId(tenantId).stream() - .map(CalculatedField::getConfiguration) - .map(CalculatedFieldConfiguration::getReferencedEntities) - .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); - } - - @Override - public boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId) { - return calculatedFieldDao.existsByEntityId(tenantId, entityId); - } - @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index 025f3dc18c..a966977968 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -41,8 +41,6 @@ public interface CalculatedFieldDao extends Dao { List removeAllByEntityId(TenantId tenantId, EntityId entityId); - boolean existsByEntityId(TenantId tenantId, EntityId entityId); - long countCFByEntityId(TenantId tenantId, EntityId entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index 904cb37e9c..8b4a5e7086 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.cf; -import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -32,8 +31,6 @@ public interface CalculatedFieldLinkDao extends Dao { List findCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); - ListenableFuture> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); - List findAll(); PageData findAll(PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index fdb46d3451..e2b21ed59b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -739,7 +739,6 @@ public class ModelConstants { public static final String CALCULATED_FIELD_LINK_ENTITY_TYPE = ENTITY_TYPE_COLUMN; public static final String CALCULATED_FIELD_LINK_ENTITY_ID = ENTITY_ID_COLUMN; public static final String CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID = "calculated_field_id"; - public static final String CALCULATED_FIELD_LINK_CONFIGURATION = "configuration"; protected static final String[] NONE_AGGREGATION_COLUMNS = new String[]{LONG_VALUE_COLUMN, DOUBLE_VALUE_COLUMN, BOOLEAN_VALUE_COLUMN, STRING_VALUE_COLUMN, JSON_VALUE_COLUMN, KEY_COLUMN, TS_COLUMN}; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 34c1933065..2668a1a0d8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -22,22 +22,17 @@ import jakarta.persistence.Entity; import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseEntity; import org.thingsboard.server.dao.model.BaseSqlEntity; -import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID; -import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_ID; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_TYPE; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_TABLE_NAME; @@ -61,24 +56,10 @@ public class CalculatedFieldLinkEntity extends BaseSqlEntity> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - return service.submit(() -> findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId)); - } - @Override public List findAll() { return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAll()); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 7fe57afd58..19551a786a 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -947,7 +947,6 @@ CREATE TABLE IF NOT EXISTS calculated_field_link ( entity_type VARCHAR(32), entity_id uuid NOT NULL, calculated_field_id uuid NOT NULL, - configuration varchar(10000), CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE ); diff --git a/pom.xml b/pom.xml index 052a2e41e7..0f26f4f05c 100755 --- a/pom.xml +++ b/pom.xml @@ -166,7 +166,7 @@ 2.19.0 9.2.0 - 9.4.0 + 9.10.0 From b2c58a519c1ea320c9c01a705dcaaa56d79f78ce Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 27 Feb 2025 16:55:33 +0200 Subject: [PATCH 23/25] added previousCalculatedFields property to delete requests --- .../controller/SystemInfoController.java | 2 ++ .../DefaultCalculatedFieldQueueService.java | 5 ++- .../server/common/data/SystemParams.java | 2 ++ .../engine/api/AttributesDeleteRequest.java | 12 +++++-- .../engine/api/AttributesSaveRequest.java | 2 +- .../CalculatedFieldSystemAwareRequest.java | 32 +++++++++++++++++++ .../engine/api/TimeseriesDeleteRequest.java | 12 +++++-- .../engine/api/TimeseriesSaveRequest.java | 2 +- .../telemetry/TbMsgDeleteAttributesNode.java | 1 + 9 files changed, 61 insertions(+), 9 deletions(-) create mode 100644 rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/CalculatedFieldSystemAwareRequest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java index 962a6f3a48..4ee86871d6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java +++ b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java @@ -156,6 +156,8 @@ public class SystemInfoController extends BaseController { if (debugModeRateLimitsConfig.isCalculatedFieldDebugPerTenantLimitsEnabled()) { systemParams.setCalculatedFieldDebugPerTenantLimitsConfiguration(debugModeRateLimitsConfig.getCalculatedFieldDebugPerTenantLimitsConfiguration()); } + systemParams.setMaxArgumentsPerCF(tenantProfileConfiguration.getMaxArgumentsPerCF()); + systemParams.setMaxDataPointsPerRollingArg(tenantProfileConfiguration.getMaxDataPointsPerRollingArg()); } systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID)) .map(QrCodeSettings::getQrCodeConfig).map(QRCodeConfig::isShowOnHomePage) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 7421dfe187..08b2011adc 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -50,7 +50,6 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; -import java.util.ArrayList; import java.util.EnumSet; import java.util.List; import java.util.Set; @@ -195,7 +194,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesDeleteRequest request, List removedKeys) { - CalculatedFieldTelemetryMsgProto telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), new ArrayList<>(), request.getTbMsgId(), request.getTbMsgType()) + CalculatedFieldTelemetryMsgProto telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()) .setScope(AttributeScopeProto.valueOf(request.getScope().name())) .addAllRemovedAttrKeys(removedKeys).build(); return ToCalculatedFieldMsg.newBuilder() @@ -204,7 +203,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesDeleteRequest request, List removedKeys) { - CalculatedFieldTelemetryMsgProto telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), new ArrayList<>(), request.getTbMsgId(), request.getTbMsgType()) + CalculatedFieldTelemetryMsgProto telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()) .addAllRemovedTsKeys(removedKeys).build(); return ToCalculatedFieldMsg.newBuilder() .setTelemetryMsg(telemetryMsg) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java index 49d9b11a35..b1ef4d7f22 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java @@ -35,4 +35,6 @@ public class SystemParams { int maxDebugModeDurationMinutes; String ruleChainDebugPerTenantLimitsConfiguration; String calculatedFieldDebugPerTenantLimitsConfiguration; + long maxArgumentsPerCF; + long maxDataPointsPerRollingArg; } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesDeleteRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesDeleteRequest.java index 77de99aae0..2ab6923899 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesDeleteRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesDeleteRequest.java @@ -22,6 +22,7 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.ToString; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.msg.TbMsgType; @@ -32,13 +33,14 @@ import java.util.UUID; @Getter @ToString @AllArgsConstructor(access = AccessLevel.PRIVATE) -public class AttributesDeleteRequest { +public class AttributesDeleteRequest implements CalculatedFieldSystemAwareRequest { private final TenantId tenantId; private final EntityId entityId; private final AttributeScope scope; private final List keys; private final boolean notifyDevice; + private final List previousCalculatedFieldIds; private final UUID tbMsgId; private final TbMsgType tbMsgType; private final FutureCallback callback; @@ -54,6 +56,7 @@ public class AttributesDeleteRequest { private AttributeScope scope; private List keys; private boolean notifyDevice; + private List previousCalculatedFieldIds; private UUID tbMsgId; private TbMsgType tbMsgType; private FutureCallback callback; @@ -96,6 +99,11 @@ public class AttributesDeleteRequest { return this; } + public Builder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; + return this; + } + public Builder tbMsgId(UUID tbMsgId) { this.tbMsgId = tbMsgId; return this; @@ -126,7 +134,7 @@ public class AttributesDeleteRequest { } public AttributesDeleteRequest build() { - return new AttributesDeleteRequest(tenantId, entityId, scope, keys, notifyDevice, tbMsgId, tbMsgType, callback); + return new AttributesDeleteRequest(tenantId, entityId, scope, keys, notifyDevice, previousCalculatedFieldIds, tbMsgId, tbMsgType, callback); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java index ab4937e952..7c8e9d317e 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java @@ -36,7 +36,7 @@ import java.util.UUID; @Getter @ToString @AllArgsConstructor(access = AccessLevel.PRIVATE) -public class AttributesSaveRequest { +public class AttributesSaveRequest implements CalculatedFieldSystemAwareRequest { private final TenantId tenantId; private final EntityId entityId; diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/CalculatedFieldSystemAwareRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/CalculatedFieldSystemAwareRequest.java new file mode 100644 index 0000000000..fa4c414172 --- /dev/null +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/CalculatedFieldSystemAwareRequest.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.api; + +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.msg.TbMsgType; + +import java.util.List; +import java.util.UUID; + +public interface CalculatedFieldSystemAwareRequest { + + List getPreviousCalculatedFieldIds(); + + UUID getTbMsgId(); + + TbMsgType getTbMsgType(); + +} diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java index 73d18268de..2ff431c928 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java @@ -20,6 +20,7 @@ import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.ToString; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; @@ -31,12 +32,13 @@ import java.util.UUID; @Getter @ToString @AllArgsConstructor(access = AccessLevel.PRIVATE) -public class TimeseriesDeleteRequest { +public class TimeseriesDeleteRequest implements CalculatedFieldSystemAwareRequest { private final TenantId tenantId; private final EntityId entityId; private final List keys; private final List deleteHistoryQueries; + private final List previousCalculatedFieldIds; private final UUID tbMsgId; private final TbMsgType tbMsgType; private final FutureCallback> callback; @@ -51,6 +53,7 @@ public class TimeseriesDeleteRequest { private EntityId entityId; private List keys; private List deleteHistoryQueries; + private List previousCalculatedFieldIds; private UUID tbMsgId; private TbMsgType tbMsgType; private FutureCallback> callback; @@ -78,6 +81,11 @@ public class TimeseriesDeleteRequest { return this; } + public Builder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; + return this; + } + public Builder tbMsgId(UUID tbMsgId) { this.tbMsgId = tbMsgId; return this; @@ -94,7 +102,7 @@ public class TimeseriesDeleteRequest { } public TimeseriesDeleteRequest build() { - return new TimeseriesDeleteRequest(tenantId, entityId, keys, deleteHistoryQueries, tbMsgId, tbMsgType, callback); + return new TimeseriesDeleteRequest(tenantId, entityId, keys, deleteHistoryQueries, previousCalculatedFieldIds, tbMsgId, tbMsgType, callback); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index 23ef568449..01af278efb 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -37,7 +37,7 @@ import static java.util.Objects.requireNonNullElse; @Getter @AllArgsConstructor(access = AccessLevel.PRIVATE) -public class TimeseriesSaveRequest { +public class TimeseriesSaveRequest implements CalculatedFieldSystemAwareRequest { private final TenantId tenantId; private final CustomerId customerId; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNode.java index 585181423d..32d700dc1d 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNode.java @@ -76,6 +76,7 @@ public class TbMsgDeleteAttributesNode implements TbNode { .scope(scope) .keys(keysToDelete) .notifyDevice(checkNotifyDevice(msg.getMetaData().getValue(NOTIFY_DEVICE_METADATA_KEY), scope)) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) .tbMsgId(msg.getId()) .tbMsgType(msg.getInternalType()) .callback(config.isSendAttributesDeletedNotification() ? From 5ca236f8fe281d90442667fd74fecf64e58d4d0f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 27 Feb 2025 18:17:03 +0200 Subject: [PATCH 24/25] Avoid start of RocksDb in Kafka mode --- .../main/java/org/thingsboard/server/service/cf/CfRocksDb.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java b/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java index 90c796f1f8..cb6d3f0d6b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java @@ -19,10 +19,12 @@ import jakarta.annotation.PreDestroy; import org.rocksdb.Options; import org.rocksdb.WriteOptions; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.utils.TbRocksDb; @Component +@ConditionalOnExpression("'${queue.type:null}'=='in-memory'") public class CfRocksDb extends TbRocksDb { public CfRocksDb(@Value("${queue.calculated_fields.rocks_db_path:${user.home}/.rocksdb/cf_states}") String path) throws Exception { From f0f2d1f29f033d5bd2c77896fb11cf98f60dbf0b Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 27 Feb 2025 18:31:35 +0200 Subject: [PATCH 25/25] Avoid start of Kafka State Service for Cores --- .../service/cf/ctx/state/KafkaCalculatedFieldStateService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index edb47dd7fa..e81fa4d1dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -49,7 +49,7 @@ import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.*; @Service @RequiredArgsConstructor @Slf4j -@ConditionalOnExpression("'${queue.type:null}'=='kafka'") +@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine') && '${queue.type:null}'=='kafka'") public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldStateService { private final TbRuleEngineQueueFactory queueFactory;