From 5563da7c0d9e48fc707626e23db0082c6bf68faa Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 10 Feb 2025 16:28:59 +0200 Subject: [PATCH 01/22] implemented filters for AlarmCountQuery --- ...efaultTbEntityDataSubscriptionService.java | 8 +- .../subscription/TbAlarmCountSubCtx.java | 37 +++++- .../server/controller/WebsocketApiTest.java | 114 +++++++++++++++++- .../server/dao/alarm/AlarmService.java | 2 + .../common/data/query/AlarmCountQuery.java | 7 ++ .../common/data/query/EntityCountQuery.java | 2 +- .../server/dao/alarm/AlarmDao.java | 2 +- .../server/dao/alarm/BaseAlarmService.java | 8 +- .../server/dao/sql/alarm/JpaAlarmDao.java | 4 +- .../dao/sql/query/AlarmQueryRepository.java | 2 +- .../query/DefaultAlarmQueryRepository.java | 11 +- .../server/dao/service/AlarmServiceTest.java | 51 ++++++++ 12 files changed, 230 insertions(+), 18 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java index 946a425a71..9812872bf1 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java @@ -33,7 +33,6 @@ import org.springframework.stereotype.Service; import org.springframework.web.socket.CloseStatus; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; @@ -41,7 +40,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.common.data.query.ComparisonTsValue; -import org.thingsboard.server.common.data.query.OriginatorAlarmFilter; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityKey; @@ -55,7 +53,6 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; -import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.ws.WebSocketService; import org.thingsboard.server.service.ws.WebSocketSessionRef; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AggHistoryCmd; @@ -65,7 +62,6 @@ import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmStatusCmd; -import org.thingsboard.server.service.ws.telemetry.cmd.v2.CmdUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataUpdate; @@ -74,7 +70,6 @@ import org.thingsboard.server.service.ws.telemetry.cmd.v2.GetTsCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.LatestValueCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.TimeSeriesCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.UnsubscribeCmd; -import org.thingsboard.server.service.ws.telemetry.sub.AlarmSubscriptionUpdate; import java.util.ArrayList; import java.util.Arrays; @@ -83,7 +78,6 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; @@ -555,7 +549,7 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc private TbAlarmCountSubCtx createSubCtx(WebSocketSessionRef sessionRef, AlarmCountCmd cmd) { Map sessionSubs = subscriptionsBySessionId.computeIfAbsent(sessionRef.getSessionId(), k -> new ConcurrentHashMap<>()); TbAlarmCountSubCtx ctx = new TbAlarmCountSubCtx(serviceId, wsService, entityService, localSubscriptionService, - attributesService, stats, alarmService, sessionRef, cmd.getCmdId()); + attributesService, stats, alarmService, sessionRef, cmd.getCmdId(), maxEntitiesPerAlarmSubscription); if (cmd.getQuery() != null) { ctx.setAndResolveQuery(cmd.getQuery()); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java index af7cfcfcfe..8425b2a57a 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java @@ -19,20 +19,33 @@ import lombok.Getter; import lombok.Setter; import lombok.ToString; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.service.ws.WebSocketService; import org.thingsboard.server.service.ws.WebSocketSessionRef; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountUpdate; +import java.util.List; + @Slf4j @ToString(callSuper = true) public class TbAlarmCountSubCtx extends TbAbstractEntityQuerySubCtx { private final AlarmService alarmService; + private final int maxEntitiesPerAlarmSubscription; + @Getter @Setter private volatile int result; @@ -40,20 +53,21 @@ public class TbAlarmCountSubCtx extends TbAbstractEntityQuerySubCtx data = entityService.findEntityDataByQuery(getTenantId(), getCustomerId(), buildEntityDataQuery()); + List entityIds = data.getData().stream().map(EntityData::getEntityId).toList(); + if (entityIds.isEmpty()) { + return 0; + } else { + return (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query, entityIds); + } + + } + + private EntityDataQuery buildEntityDataQuery() { + EntityDataPageLink edpl = new EntityDataPageLink(maxEntitiesPerAlarmSubscription, 0, null, + new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, ModelConstants.CREATED_TIME_PROPERTY))); + return new EntityDataQuery(query.getEntityFilter(), edpl, null, null, null); + } } diff --git a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java index b64c87ccbd..e72e1dc806 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java @@ -83,7 +83,8 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @Slf4j @DaoSqlTest @TestPropertySource(properties = { - "server.ws.alarms_per_alarm_status_subscription_cache_size=5" + "server.ws.alarms_per_alarm_status_subscription_cache_size=5", + "server.ws.dynamic_page_link.refresh_interval=2" }) public class WebsocketApiTest extends AbstractControllerTest { @Autowired @@ -324,6 +325,117 @@ public class WebsocketApiTest extends AbstractControllerTest { Assert.assertEquals(1, update.getCount()); } + @Test + public void testAlarmCountWsCmdWithSingleEntityFilter() throws Exception { + loginTenantAdmin(); + + SingleEntityFilter singleEntityFilter = new SingleEntityFilter(); + singleEntityFilter.setSingleEntity(tenantId); + AlarmCountQuery alarmCountQuery = new AlarmCountQuery(singleEntityFilter); + AlarmCountCmd cmd1 = new AlarmCountCmd(1, alarmCountQuery); + + getWsClient().send(cmd1); + + AlarmCountUpdate update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(1, update.getCmdId()); + Assert.assertEquals(0, update.getCount()); + + Alarm alarm = new Alarm(); + alarm.setOriginator(tenantId); + alarm.setType("TEST ALARM"); + alarm.setSeverity(AlarmSeverity.WARNING); + + alarm = doPost("/api/alarm", alarm, Alarm.class); + + AlarmCountCmd cmd2 = new AlarmCountCmd(2, alarmCountQuery); + + getWsClient().send(cmd2); + + update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(2, update.getCmdId()); + Assert.assertEquals(1, update.getCount()); + + singleEntityFilter.setSingleEntity(tenantAdminUserId); + AlarmCountCmd cmd3 = new AlarmCountCmd(3, alarmCountQuery); + + getWsClient().send(cmd3); + + update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(3, update.getCmdId()); + Assert.assertEquals(0, update.getCount()); + + alarm.setAssigneeId(tenantAdminUserId); + alarm = doPost("/api/alarm", alarm, Alarm.class); + + singleEntityFilter.setSingleEntity(tenantId); + alarmCountQuery.setAssigneeId(tenantAdminUserId); + AlarmCountCmd cmd4 = new AlarmCountCmd(4, alarmCountQuery); + + getWsClient().send(cmd4); + + update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(4, update.getCmdId()); + Assert.assertEquals(1, update.getCount()); + + alarmCountQuery.setSeverityList(Collections.singletonList(AlarmSeverity.CRITICAL)); + AlarmCountCmd cmd5 = new AlarmCountCmd(5, alarmCountQuery); + + getWsClient().send(cmd5); + + update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(5, update.getCmdId()); + Assert.assertEquals(0, update.getCount()); + + alarm.setSeverity(AlarmSeverity.CRITICAL); + doPost("/api/alarm", alarm, Alarm.class); + + AlarmCountCmd cmd6 = new AlarmCountCmd(6, alarmCountQuery); + + getWsClient().send(cmd6); + + update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(6, update.getCmdId()); + Assert.assertEquals(1, update.getCount()); + } + + @Test + public void testAlarmCountWsCmdWithDeviceType() throws Exception { + loginTenantAdmin(); + + DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter(); + deviceTypeFilter.setDeviceTypes(List.of("default")); + AlarmCountQuery alarmCountQuery = new AlarmCountQuery(deviceTypeFilter); + AlarmCountCmd cmd1 = new AlarmCountCmd(1, alarmCountQuery); + + getWsClient().send(cmd1); + + AlarmCountUpdate update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(1, update.getCmdId()); + Assert.assertEquals(0, update.getCount()); + + getWsClient().registerWaitForUpdate(); + + Alarm alarm = new Alarm(); + alarm.setOriginator(device.getId()); + alarm.setType("TEST ALARM"); + alarm.setSeverity(AlarmSeverity.WARNING); + + alarm = doPost("/api/alarm", alarm, Alarm.class); + + update = getWsClient().parseAlarmCountReply(getWsClient().waitForUpdate(3000)); + Assert.assertEquals(1, update.getCmdId()); + Assert.assertEquals(1, update.getCount()); + + deviceTypeFilter.setDeviceTypes(List.of("non-existing")); + AlarmCountCmd cmd3 = new AlarmCountCmd(3, alarmCountQuery); + + getWsClient().send(cmd3); + + update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(3, update.getCmdId()); + Assert.assertEquals(0, update.getCount()); + } + @Test public void testAlarmStatusWsCmd() throws Exception { loginTenantAdmin(); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index ce040595cc..11252b59a5 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -118,6 +118,8 @@ public interface AlarmService extends EntityDaoService { long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query); + long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection orderedEntityIds); + PageData findAlarmTypesByTenantId(TenantId tenantId, PageLink pageLink); List findActiveOriginatorAlarms(TenantId tenantId, OriginatorAlarmFilter originatorAlarmFilter, int limit); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java index 19cb445380..bb969a28ce 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java @@ -19,6 +19,7 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; +import lombok.Setter; import lombok.ToString; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; import org.thingsboard.server.common.data.alarm.AlarmSeverity; @@ -30,6 +31,7 @@ import java.util.List; @NoArgsConstructor @AllArgsConstructor @Getter +@Setter @ToString public class AlarmCountQuery extends EntityCountQuery { private long startTs; @@ -40,4 +42,9 @@ public class AlarmCountQuery extends EntityCountQuery { private List severityList; private boolean searchPropagatedAlarms; private UserId assigneeId; + + public AlarmCountQuery(EntityFilter entityFilter) { + super(entityFilter); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java index 3dd24147e7..be9fa4c664 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java @@ -27,7 +27,7 @@ import java.util.List; public class EntityCountQuery { @Getter - private EntityFilter entityFilter; + protected EntityFilter entityFilter; @Getter protected List keyFilters; diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java index 4b4ee0ae4f..2432bb4bbe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java @@ -106,7 +106,7 @@ public interface AlarmDao extends Dao { AlarmApiCallResult unassignAlarm(TenantId tenantId, AlarmId alarmId, long unassignTime); - long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query); + long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection orderedEntityIds); PageData findTenantAlarmTypes(UUID tenantId, PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index bf08621d14..14cd65185d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -352,7 +352,13 @@ public class BaseAlarmService extends AbstractCachedEntityService INCORRECT_TENANT_ID + id); - return alarmDao.countAlarmsByQuery(tenantId, customerId, query); + return alarmDao.countAlarmsByQuery(tenantId, customerId, query, Collections.emptyList()); + } + + @Override + public long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection orderedEntityIds) { + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + return alarmDao.countAlarmsByQuery(tenantId, customerId, query, orderedEntityIds); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index e433dfbd86..a3e02d5461 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -415,8 +415,8 @@ public class JpaAlarmDao extends JpaAbstractDao implements A } @Override - public long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query) { - return alarmQueryRepository.countAlarmsByQuery(tenantId, customerId, query); + public long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection orderedEntityIds) { + return alarmQueryRepository.countAlarmsByQuery(tenantId, customerId, query, orderedEntityIds); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmQueryRepository.java index 49aa6c7276..2a26d62376 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmQueryRepository.java @@ -30,6 +30,6 @@ public interface AlarmQueryRepository { PageData findAlarmDataByQueryForEntities(TenantId tenantId, AlarmDataQuery query, Collection orderedEntityIds); - long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query); + long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection orderedEntityIds); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index 05da22f9a5..b4cc40fc6d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -44,6 +44,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.UUID; import java.util.stream.Collectors; @Repository @@ -314,7 +315,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { } @Override - public long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query) { + public long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection orderedEntityIds) { QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); if (query.isSearchPropagatedAlarms()) { @@ -326,6 +327,10 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { ctx.append(" and a.customer_id = :customerId and ea.customer_id = :customerId"); ctx.addUuidParameter("customerId", customerId.getId()); } + if (!orderedEntityIds.isEmpty()) { + ctx.addUuidListParameter("entity_filter_entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList())); + ctx.append(" and ea.entity_id in (:entity_filter_entity_ids)"); + } } else { ctx.append("select count(id) from alarm_info a "); ctx.append("where a.tenant_id = :tenantId"); @@ -334,6 +339,10 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { ctx.append(" and a.customer_id = :customerId"); ctx.addUuidParameter("customerId", customerId.getId()); } + if (!orderedEntityIds.isEmpty()) { + ctx.addUuidListParameter("entity_filter_entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList())); + ctx.append(" and a.originator_id in (:entity_filter_entity_ids)"); + } } long startTs; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AlarmServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AlarmServiceTest.java index aa1c96d84b..d12765717a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AlarmServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AlarmServiceTest.java @@ -22,6 +22,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmApiCallResult; @@ -48,6 +49,7 @@ import org.thingsboard.server.common.data.query.DeviceTypeFilter; import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.security.Authority; @@ -936,4 +938,53 @@ public class AlarmServiceTest extends AbstractServiceTest { Assert.assertEquals(0, alarms.getData().size()); } + @Test + public void testCountAlarmsForEntities() throws ExecutionException, InterruptedException { + AssetId parentId = new AssetId(Uuids.timeBased()); + AssetId childId = new AssetId(Uuids.timeBased()); + + EntityRelation relation = new EntityRelation(parentId, childId, EntityRelation.CONTAINS_TYPE); + + Assert.assertTrue(relationService.saveRelationAsync(tenantId, relation).get()); + + long ts = System.currentTimeMillis(); + AlarmApiCallResult result = alarmService.createAlarm(AlarmCreateOrUpdateActiveRequest.builder() + .tenantId(tenantId) + .originator(childId) + .type(TEST_ALARM) + .severity(AlarmSeverity.CRITICAL) + .startTs(ts).build()); + AlarmInfo created = result.getAlarm(); + created.setPropagate(true); + result = alarmService.updateAlarm(AlarmUpdateRequest.fromAlarm(created)); + created = result.getAlarm(); + + EntityListFilter entityListFilter = new EntityListFilter(); + entityListFilter.setEntityList(List.of(childId.getId().toString(), parentId.getId().toString())); + entityListFilter.setEntityType(EntityType.ASSET); + AlarmCountQuery countQuery = new AlarmCountQuery(entityListFilter); + countQuery.setStartTs(0L); + countQuery.setEndTs(System.currentTimeMillis()); + + long alarmsCount = alarmService.countAlarmsByQuery(tenantId, null, countQuery, List.of(childId)); + Assert.assertEquals(1, alarmsCount); + + countQuery.setSearchPropagatedAlarms(true); + + alarmsCount = alarmService.countAlarmsByQuery(tenantId, null, countQuery, List.of(parentId)); + Assert.assertEquals(1, alarmsCount); + + created = alarmService.acknowledgeAlarm(tenantId, created.getId(), System.currentTimeMillis()).getAlarm(); + + countQuery.setStatusList(List.of(AlarmSearchStatus.UNACK)); + alarmsCount = alarmService.countAlarmsByQuery(tenantId, null, countQuery, List.of(childId)); + Assert.assertEquals(0, alarmsCount); + + alarmService.clearAlarm(tenantId, created.getId(), System.currentTimeMillis(), null); + + countQuery.setStatusList(List.of(AlarmSearchStatus.CLEARED)); + alarmsCount = alarmService.countAlarmsByQuery(tenantId, null, countQuery, List.of(childId)); + Assert.assertEquals(1, alarmsCount); + } + } From 33eac2f77868c4b79c1beb2a99a02fa0047f756c Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 10 Feb 2025 17:34:12 +0200 Subject: [PATCH 02/22] fixed tests --- .../service/subscription/TbAlarmCountSubCtx.java | 15 ++++++++------- .../server/common/data/query/AlarmCountQuery.java | 6 ++---- .../common/data/query/EntityCountQuery.java | 2 +- .../server/dao/alarm/BaseAlarmService.java | 3 +-- .../sql/query/DefaultAlarmQueryRepository.java | 4 ++-- 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java index 8425b2a57a..4771bad716 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java @@ -80,14 +80,15 @@ public class TbAlarmCountSubCtx extends TbAbstractEntityQuerySubCtx data = entityService.findEntityDataByQuery(getTenantId(), getCustomerId(), buildEntityDataQuery()); - List entityIds = data.getData().stream().map(EntityData::getEntityId).toList(); - if (entityIds.isEmpty()) { - return 0; - } else { - return (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query, entityIds); + List entityIds = null; + if (query.getEntityFilter() != null) { + PageData data = entityService.findEntityDataByQuery(getTenantId(), getCustomerId(), buildEntityDataQuery()); + if (data.getData().isEmpty()) { + return 0; + } + entityIds = data.getData().stream().map(EntityData::getEntityId).toList(); } - + return (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query, entityIds); } private EntityDataQuery buildEntityDataQuery() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java index bb969a28ce..1bf19a970f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java @@ -17,9 +17,8 @@ package org.thingsboard.server.common.data.query; import lombok.AllArgsConstructor; import lombok.Builder; -import lombok.Getter; +import lombok.Data; import lombok.NoArgsConstructor; -import lombok.Setter; import lombok.ToString; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; import org.thingsboard.server.common.data.alarm.AlarmSeverity; @@ -30,8 +29,7 @@ import java.util.List; @Builder @NoArgsConstructor @AllArgsConstructor -@Getter -@Setter +@Data @ToString public class AlarmCountQuery extends EntityCountQuery { private long startTs; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java index be9fa4c664..3dd24147e7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java @@ -27,7 +27,7 @@ import java.util.List; public class EntityCountQuery { @Getter - protected EntityFilter entityFilter; + private EntityFilter entityFilter; @Getter protected List keyFilters; diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index 14cd65185d..b8a57fdcff 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -351,8 +351,7 @@ public class BaseAlarmService extends AbstractCachedEntityService INCORRECT_TENANT_ID + id); - return alarmDao.countAlarmsByQuery(tenantId, customerId, query, Collections.emptyList()); + return countAlarmsByQuery(tenantId, customerId, query, null); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index b4cc40fc6d..216fd0d2d4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -327,7 +327,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { ctx.append(" and a.customer_id = :customerId and ea.customer_id = :customerId"); ctx.addUuidParameter("customerId", customerId.getId()); } - if (!orderedEntityIds.isEmpty()) { + if (orderedEntityIds != null) { ctx.addUuidListParameter("entity_filter_entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList())); ctx.append(" and ea.entity_id in (:entity_filter_entity_ids)"); } @@ -339,7 +339,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { ctx.append(" and a.customer_id = :customerId"); ctx.addUuidParameter("customerId", customerId.getId()); } - if (!orderedEntityIds.isEmpty()) { + if (orderedEntityIds != null) { ctx.addUuidListParameter("entity_filter_entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList())); ctx.append(" and a.originator_id in (:entity_filter_entity_ids)"); } From 7b6bb8a6d61546a82305c51862ee087116b10827 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 12 Feb 2025 12:34:56 +0200 Subject: [PATCH 03/22] UI: Add support for entity alias filtering in alarm count data sources --- .../server/service/subscription/TbAlarmCountSubCtx.java | 2 +- .../basic/alarm/alarm-count-basic-config.component.html | 9 +++++++++ .../basic/alarm/alarm-count-basic-config.component.ts | 2 ++ .../components/widget/config/datasource.component.html | 8 ++++---- .../components/widget/config/datasource.component.ts | 9 +++++++++ .../components/widget/config/datasources.component.ts | 4 ++++ 6 files changed, 29 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java index 4771bad716..2a5dea70ac 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java @@ -94,6 +94,6 @@ public class TbAlarmCountSubCtx extends TbAbstractEntityQuerySubCtx + +
alarm.filter
diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/alarm/alarm-count-basic-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/alarm/alarm-count-basic-config.component.ts index 4ad664fae3..d365ed3f2d 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/alarm/alarm-count-basic-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/alarm/alarm-count-basic-config.component.ts @@ -69,6 +69,7 @@ export class AlarmCountBasicConfigComponent extends BasicWidgetConfigComponent { const settings: CountWidgetSettings = {...countDefaultSettings(true), ...(configData.config.settings || {})}; this.alarmCountWidgetConfigForm = this.fb.group({ alarmFilterConfig: [getAlarmFilterConfig(configData.config.datasources), []], + datasources: [configData.config.datasources, []], settings: [settings, []], @@ -81,6 +82,7 @@ export class AlarmCountBasicConfigComponent extends BasicWidgetConfigComponent { } protected prepareOutputConfig(config: any): WidgetConfigComponentData { + this.widgetConfig.config.datasources = config.datasources; setAlarmFilterConfig(config.alarmFilterConfig, this.widgetConfig.config.datasources); this.widgetConfig.config.settings = {...(this.widgetConfig.config.settings || {}), ...config.settings}; diff --git a/ui-ngx/src/app/modules/home/components/widget/config/datasource.component.html b/ui-ngx/src/app/modules/home/components/widget/config/datasource.component.html index 8af7753525..6c5c5cf083 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/datasource.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/config/datasource.component.html @@ -36,7 +36,7 @@ datasourceFormGroup.get('type').value === datasourceType.entity || datasourceFormGroup.get('type').value === datasourceType.entityCount || datasourceFormGroup.get('type').value === datasourceType.alarmCount ? datasourceFormGroup.get('type').value : ''"> - @@ -98,7 +98,7 @@ Date: Wed, 12 Feb 2025 15:38:25 +0200 Subject: [PATCH 04/22] fetch entities only on subs creation --- ...efaultTbEntityDataSubscriptionService.java | 10 +- .../subscription/TbAlarmCountSubCtx.java | 106 ++++++++++++++---- .../server/controller/WebsocketApiTest.java | 4 +- .../query/DefaultAlarmQueryRepository.java | 36 +++--- 4 files changed, 117 insertions(+), 39 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java index 9812872bf1..b09f8350f1 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java @@ -424,8 +424,12 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc long start = System.currentTimeMillis(); ctx.fetchData(); long end = System.currentTimeMillis(); - stats.getAlarmQueryInvocationCnt().incrementAndGet(); - stats.getAlarmQueryTimeSpent().addAndGet(end - start); + stats.getRegularQueryInvocationCnt().incrementAndGet(); + stats.getRegularQueryTimeSpent().addAndGet(end - start); + ctx.cancelTasks(); + ctx.clearAlarmSubscriptions(); + ctx.fetchAlarmCount(); + ctx.createAlarmSubscriptions(); TbAlarmCountSubCtx finalCtx = ctx; ScheduledFuture task = scheduler.scheduleWithFixedDelay( () -> refreshDynamicQuery(finalCtx), @@ -549,7 +553,7 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc private TbAlarmCountSubCtx createSubCtx(WebSocketSessionRef sessionRef, AlarmCountCmd cmd) { Map sessionSubs = subscriptionsBySessionId.computeIfAbsent(sessionRef.getSessionId(), k -> new ConcurrentHashMap<>()); TbAlarmCountSubCtx ctx = new TbAlarmCountSubCtx(serviceId, wsService, entityService, localSubscriptionService, - attributesService, stats, alarmService, sessionRef, cmd.getCmdId(), maxEntitiesPerAlarmSubscription); + attributesService, stats, alarmService, sessionRef, cmd.getCmdId(), maxEntitiesPerAlarmSubscription, maxAlarmQueriesPerRefreshInterval); if (cmd.getQuery() != null) { ctx.setAndResolveQuery(cmd.getQuery()); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java index 2a5dea70ac..712fc5c378 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java @@ -36,7 +36,9 @@ import org.thingsboard.server.service.ws.WebSocketService; import org.thingsboard.server.service.ws.WebSocketSessionRef; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountUpdate; -import java.util.List; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; @Slf4j @ToString(callSuper = true) @@ -44,56 +46,120 @@ public class TbAlarmCountSubCtx extends TbAbstractEntityQuerySubCtx subToEntityIdMap; + + private LinkedHashSet entitiesIds; + private final int maxEntitiesPerAlarmSubscription; + private final int maxAlarmQueriesPerRefreshInterval; + @Getter @Setter private volatile int result; + @Getter + @Setter + private boolean tooManyEntities; + + private int alarmCountInvocationAttempts; + public TbAlarmCountSubCtx(String serviceId, WebSocketService wsService, EntityService entityService, TbLocalSubscriptionService localSubscriptionService, AttributesService attributesService, SubscriptionServiceStatistics stats, AlarmService alarmService, - WebSocketSessionRef sessionRef, int cmdId, int maxEntitiesPerAlarmSubscription) { + WebSocketSessionRef sessionRef, int cmdId, int maxEntitiesPerAlarmSubscription, int maxAlarmQueriesPerRefreshInterval) { super(serviceId, wsService, entityService, localSubscriptionService, attributesService, stats, sessionRef, cmdId); this.alarmService = alarmService; + this.subToEntityIdMap = new ConcurrentHashMap<>(); this.maxEntitiesPerAlarmSubscription = maxEntitiesPerAlarmSubscription; + this.maxAlarmQueriesPerRefreshInterval = maxAlarmQueriesPerRefreshInterval; + this.entitiesIds = null; + } + + @Override + public void clearSubscriptions() { + clearAlarmSubscriptions(); } @Override public void fetchData() { - result = countAlarms(); - sendWsMsg(new AlarmCountUpdate(cmdId, result)); + resetInvocationCounter(); + if (query.getEntityFilter() != null) { + entitiesIds = new LinkedHashSet<>(); + log.trace("[{}] Fetching data: {}", cmdId, alarmCountInvocationAttempts); + PageData data = entityService.findEntityDataByQuery(getTenantId(), getCustomerId(), buildEntityDataQuery()); + entitiesIds.clear(); + tooManyEntities = data.hasNext(); + for (EntityData entityData : data.getData()) { + entitiesIds.add(entityData.getEntityId()); + } + } } @Override protected void update() { - int newCount = countAlarms(); - if (newCount != result) { - result = newCount; - sendWsMsg(new AlarmCountUpdate(cmdId, result)); + resetInvocationCounter(); + fetchAlarmCount(); + } + + public void fetchAlarmCount() { + alarmCountInvocationAttempts++; + log.trace("[{}] Fetching alarms: {}", cmdId, alarmCountInvocationAttempts); + if (alarmCountInvocationAttempts <= maxAlarmQueriesPerRefreshInterval) { + doFetchAlarmCount(); + } else { + log.trace("[{}] Ignore alarm count fetch due to rate limit: [{}] of maximum [{}]", cmdId, alarmCountInvocationAttempts, maxAlarmQueriesPerRefreshInterval); } } + private void doFetchAlarmCount() { + result = (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query, entitiesIds); + sendWsMsg(new AlarmCountUpdate(cmdId, result)); + } + @Override public boolean isDynamic() { return true; } - private int countAlarms() { - List entityIds = null; - if (query.getEntityFilter() != null) { - PageData data = entityService.findEntityDataByQuery(getTenantId(), getCustomerId(), buildEntityDataQuery()); - if (data.getData().isEmpty()) { - return 0; - } - entityIds = data.getData().stream().map(EntityData::getEntityId).toList(); - } - return (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query, entityIds); - } - private EntityDataQuery buildEntityDataQuery() { EntityDataPageLink edpl = new EntityDataPageLink(maxEntitiesPerAlarmSubscription, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, ModelConstants.CREATED_TIME_PROPERTY))); return new EntityDataQuery(query.getEntityFilter(), edpl, null, null, query.getKeyFilters()); } + + private void resetInvocationCounter() { + alarmCountInvocationAttempts = 0; + } + + public void createAlarmSubscriptions() { + for (EntityId entityId : entitiesIds) { + createAlarmSubscriptionForEntity(entityId); + } + } + + private void createAlarmSubscriptionForEntity(EntityId entityId) { + int subIdx = sessionRef.getSessionSubIdSeq().incrementAndGet(); + subToEntityIdMap.put(subIdx, entityId); + log.trace("[{}][{}][{}] Creating alarms subscription for [{}] ", serviceId, cmdId, subIdx, entityId); + TbAlarmsSubscription subscription = TbAlarmsSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionRef.getSessionId()) + .subscriptionId(subIdx) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .updateProcessor((sub, update) -> fetchAlarmCount()) + .build(); + localSubscriptionService.addSubscription(subscription, sessionRef); + } + + public void clearAlarmSubscriptions() { + if (subToEntityIdMap != null) { + for (Integer subId : subToEntityIdMap.keySet()) { + localSubscriptionService.cancelSubscription(getTenantId(), getSessionId(), subId); + } + subToEntityIdMap.clear(); + } + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java index e72e1dc806..d1b8bc0430 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java @@ -84,7 +84,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @DaoSqlTest @TestPropertySource(properties = { "server.ws.alarms_per_alarm_status_subscription_cache_size=5", - "server.ws.dynamic_page_link.refresh_interval=2" + "server.ws.dynamic_page_link.refresh_interval=3" }) public class WebsocketApiTest extends AbstractControllerTest { @Autowired @@ -422,7 +422,7 @@ public class WebsocketApiTest extends AbstractControllerTest { alarm = doPost("/api/alarm", alarm, Alarm.class); - update = getWsClient().parseAlarmCountReply(getWsClient().waitForUpdate(3000)); + update = getWsClient().parseAlarmCountReply(getWsClient().waitForUpdate(4000)); Assert.assertEquals(1, update.getCmdId()); Assert.assertEquals(1, update.getCount()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index 216fd0d2d4..db8c8ea552 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -321,27 +321,35 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { if (query.isSearchPropagatedAlarms()) { ctx.append("select count(distinct(a.id)) from alarm_info a "); ctx.append(JOIN_ENTITY_ALARMS); - ctx.append("where a.tenant_id = :tenantId and ea.tenant_id = :tenantId"); - ctx.addUuidParameter("tenantId", tenantId.getId()); - if (customerId != null && !customerId.isNullUid()) { - ctx.append(" and a.customer_id = :customerId and ea.customer_id = :customerId"); - ctx.addUuidParameter("customerId", customerId.getId()); - } if (orderedEntityIds != null) { + if (orderedEntityIds.isEmpty()) { + return 0; + } ctx.addUuidListParameter("entity_filter_entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList())); - ctx.append(" and ea.entity_id in (:entity_filter_entity_ids)"); + ctx.append("where ea.entity_id in (:entity_filter_entity_ids)"); + } else { + ctx.append("where a.tenant_id = :tenantId and ea.tenant_id = :tenantId"); + ctx.addUuidParameter("tenantId", tenantId.getId()); + if (customerId != null && !customerId.isNullUid()) { + ctx.append(" and a.customer_id = :customerId and ea.customer_id = :customerId"); + ctx.addUuidParameter("customerId", customerId.getId()); + } } } else { ctx.append("select count(id) from alarm_info a "); - ctx.append("where a.tenant_id = :tenantId"); - ctx.addUuidParameter("tenantId", tenantId.getId()); - if (customerId != null && !customerId.isNullUid()) { - ctx.append(" and a.customer_id = :customerId"); - ctx.addUuidParameter("customerId", customerId.getId()); - } if (orderedEntityIds != null) { + if (orderedEntityIds.isEmpty()) { + return 0; + } ctx.addUuidListParameter("entity_filter_entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList())); - ctx.append(" and a.originator_id in (:entity_filter_entity_ids)"); + ctx.append("where a.originator_id in (:entity_filter_entity_ids)"); + } else { + ctx.append("where a.tenant_id = :tenantId"); + ctx.addUuidParameter("tenantId", tenantId.getId()); + if (customerId != null && !customerId.isNullUid()) { + ctx.append(" and a.customer_id = :customerId"); + ctx.addUuidParameter("customerId", customerId.getId()); + } } } From ca1185de541197d0276c5478f3abc59743aa32ae Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 12 Feb 2025 17:02:49 +0200 Subject: [PATCH 05/22] do not send update if count was not changed --- ...efaultTbEntityDataSubscriptionService.java | 22 +++++++++++++------ .../subscription/TbAlarmCountSubCtx.java | 19 ++++++++++------ 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java index b09f8350f1..968711fa55 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java @@ -33,6 +33,7 @@ import org.springframework.stereotype.Service; import org.springframework.web.socket.CloseStatus; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; @@ -59,6 +60,7 @@ import org.thingsboard.server.service.ws.telemetry.cmd.v2.AggHistoryCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AggKey; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AggTimeSeriesCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountCmd; +import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmStatusCmd; @@ -426,15 +428,21 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc long end = System.currentTimeMillis(); stats.getRegularQueryInvocationCnt().incrementAndGet(); stats.getRegularQueryTimeSpent().addAndGet(end - start); + Set entitiesIds = ctx.getEntitiesIds(); ctx.cancelTasks(); ctx.clearAlarmSubscriptions(); - ctx.fetchAlarmCount(); - ctx.createAlarmSubscriptions(); - TbAlarmCountSubCtx finalCtx = ctx; - ScheduledFuture task = scheduler.scheduleWithFixedDelay( - () -> refreshDynamicQuery(finalCtx), - dynamicPageLinkRefreshInterval, dynamicPageLinkRefreshInterval, TimeUnit.SECONDS); - finalCtx.setRefreshTask(task); + if (entitiesIds != null && entitiesIds.isEmpty()) { + AlarmCountUpdate update = new AlarmCountUpdate(cmd.getCmdId(), 0); + ctx.sendWsMsg(update); + } else { + ctx.doFetchAlarmCount(); + ctx.createAlarmSubscriptions(); + TbAlarmCountSubCtx finalCtx = ctx; + ScheduledFuture task = scheduler.scheduleWithFixedDelay( + () -> refreshDynamicQuery(finalCtx), + dynamicPageLinkRefreshInterval, dynamicPageLinkRefreshInterval, TimeUnit.SECONDS); + finalCtx.setRefreshTask(task); + } } else { log.debug("[{}][{}] Received duplicate command: {}", session.getSessionId(), cmd.getCmdId(), cmd); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java index 712fc5c378..d0144b8787 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java @@ -48,6 +48,7 @@ public class TbAlarmCountSubCtx extends TbAbstractEntityQuerySubCtx subToEntityIdMap; + @Getter private LinkedHashSet entitiesIds; private final int maxEntitiesPerAlarmSubscription; @@ -102,26 +103,30 @@ public class TbAlarmCountSubCtx extends TbAbstractEntityQuerySubCtx Date: Thu, 13 Feb 2025 11:28:42 +0200 Subject: [PATCH 06/22] fixed flaky tests --- .../server/controller/WebsocketApiTest.java | 45 +++---------------- 1 file changed, 7 insertions(+), 38 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java index d1b8bc0430..f8a9db43d0 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java @@ -83,8 +83,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @Slf4j @DaoSqlTest @TestPropertySource(properties = { - "server.ws.alarms_per_alarm_status_subscription_cache_size=5", - "server.ws.dynamic_page_link.refresh_interval=3" + "server.ws.alarms_per_alarm_status_subscription_cache_size=5" }) public class WebsocketApiTest extends AbstractControllerTest { @Autowired @@ -340,6 +339,7 @@ public class WebsocketApiTest extends AbstractControllerTest { Assert.assertEquals(1, update.getCmdId()); Assert.assertEquals(0, update.getCount()); + //create alarm, check count = 1 Alarm alarm = new Alarm(); alarm.setOriginator(tenantId); alarm.setType("TEST ALARM"); @@ -355,6 +355,7 @@ public class WebsocketApiTest extends AbstractControllerTest { Assert.assertEquals(2, update.getCmdId()); Assert.assertEquals(1, update.getCount()); + // set wrong entity id in filter, check count = 0 singleEntityFilter.setSingleEntity(tenantAdminUserId); AlarmCountCmd cmd3 = new AlarmCountCmd(3, alarmCountQuery); @@ -363,39 +364,6 @@ public class WebsocketApiTest extends AbstractControllerTest { update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); Assert.assertEquals(3, update.getCmdId()); Assert.assertEquals(0, update.getCount()); - - alarm.setAssigneeId(tenantAdminUserId); - alarm = doPost("/api/alarm", alarm, Alarm.class); - - singleEntityFilter.setSingleEntity(tenantId); - alarmCountQuery.setAssigneeId(tenantAdminUserId); - AlarmCountCmd cmd4 = new AlarmCountCmd(4, alarmCountQuery); - - getWsClient().send(cmd4); - - update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); - Assert.assertEquals(4, update.getCmdId()); - Assert.assertEquals(1, update.getCount()); - - alarmCountQuery.setSeverityList(Collections.singletonList(AlarmSeverity.CRITICAL)); - AlarmCountCmd cmd5 = new AlarmCountCmd(5, alarmCountQuery); - - getWsClient().send(cmd5); - - update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); - Assert.assertEquals(5, update.getCmdId()); - Assert.assertEquals(0, update.getCount()); - - alarm.setSeverity(AlarmSeverity.CRITICAL); - doPost("/api/alarm", alarm, Alarm.class); - - AlarmCountCmd cmd6 = new AlarmCountCmd(6, alarmCountQuery); - - getWsClient().send(cmd6); - - update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); - Assert.assertEquals(6, update.getCmdId()); - Assert.assertEquals(1, update.getCount()); } @Test @@ -422,7 +390,7 @@ public class WebsocketApiTest extends AbstractControllerTest { alarm = doPost("/api/alarm", alarm, Alarm.class); - update = getWsClient().parseAlarmCountReply(getWsClient().waitForUpdate(4000)); + update = getWsClient().parseAlarmCountReply(getWsClient().waitForUpdate()); Assert.assertEquals(1, update.getCmdId()); Assert.assertEquals(1, update.getCount()); @@ -484,17 +452,18 @@ public class WebsocketApiTest extends AbstractControllerTest { doPost("/api/alarm", alarm2, Alarm.class); - AlarmStatusUpdate alarmStatusUpdate3 = JacksonUtil.fromString(getWsClient().waitForReply(), AlarmStatusUpdate.class); + AlarmStatusUpdate alarmStatusUpdate3 = JacksonUtil.fromString(getWsClient().waitForUpdate(), AlarmStatusUpdate.class); Assert.assertEquals(1, alarmStatusUpdate3.getCmdId()); Assert.assertTrue(alarmStatusUpdate3.isActive()); //change severity + getWsClient().registerWaitForUpdate(); alarm2.setSeverity(AlarmSeverity.MAJOR); Alarm updatedAlarm = doPost("/api/alarm", alarm2, Alarm.class); Assert.assertNotNull(updatedAlarm); Assert.assertEquals(AlarmSeverity.MAJOR, updatedAlarm.getSeverity()); - AlarmStatusUpdate alarmStatusUpdate4 = JacksonUtil.fromString(getWsClient().waitForReply(), AlarmStatusUpdate.class); + AlarmStatusUpdate alarmStatusUpdate4 = JacksonUtil.fromString(getWsClient().waitForUpdate(), AlarmStatusUpdate.class); Assert.assertEquals(1, alarmStatusUpdate4.getCmdId()); Assert.assertFalse(alarmStatusUpdate4.isActive()); From 1f3c7acc51a8b9d979150e40a76f10b85b6365f7 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 14 Feb 2025 19:01:58 +0200 Subject: [PATCH 07/22] fixed flaky test --- .../server/controller/WebsocketApiTest.java | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java index f8a9db43d0..51c9b08155 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java @@ -340,29 +340,26 @@ public class WebsocketApiTest extends AbstractControllerTest { Assert.assertEquals(0, update.getCount()); //create alarm, check count = 1 + getWsClient().registerWaitForUpdate(); + Alarm alarm = new Alarm(); alarm.setOriginator(tenantId); alarm.setType("TEST ALARM"); alarm.setSeverity(AlarmSeverity.WARNING); - alarm = doPost("/api/alarm", alarm, Alarm.class); - AlarmCountCmd cmd2 = new AlarmCountCmd(2, alarmCountQuery); - - getWsClient().send(cmd2); - - update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); - Assert.assertEquals(2, update.getCmdId()); + update = getWsClient().parseAlarmCountReply(getWsClient().waitForUpdate()); + Assert.assertEquals(1, update.getCmdId()); Assert.assertEquals(1, update.getCount()); // set wrong entity id in filter, check count = 0 singleEntityFilter.setSingleEntity(tenantAdminUserId); - AlarmCountCmd cmd3 = new AlarmCountCmd(3, alarmCountQuery); + AlarmCountCmd cmd3 = new AlarmCountCmd(2, alarmCountQuery); getWsClient().send(cmd3); update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); - Assert.assertEquals(3, update.getCmdId()); + Assert.assertEquals(2, update.getCmdId()); Assert.assertEquals(0, update.getCount()); } From 5372f22b8a314e84823d39cb4a19a762f206cf59 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 28 Feb 2025 17:08:24 +0200 Subject: [PATCH 08/22] Added latest debug event call in Calculated Field Dialog on Test function --- .../core/http/calculated-fields.service.ts | 5 +++++ .../calculated-field-dialog.component.ts | 22 ++++++++++++++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index fe5b0f7b52..66c0cb609e 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -23,6 +23,7 @@ import { CalculatedField, CalculatedFieldTestScriptInputParams } from '@shared/m import { PageLink } from '@shared/models/page/page-link'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityTestScriptResult } from '@shared/models/entity.models'; +import { CalculatedFieldEventBody } from '@shared/models/event.models'; @Injectable({ providedIn: 'root' @@ -53,4 +54,8 @@ export class CalculatedFieldsService { public testScript(inputParams: CalculatedFieldTestScriptInputParams, config?: RequestConfig): Observable { return this.http.post('/api/calculatedField/testScript', inputParams, defaultHttpOptionsFromConfig(config)); } + + public getLatestCalculatedFieldDebugEvent(id: string, config?: RequestConfig): Observable { + return this.http.get(`/api/calculatedField/${id}/debug`, defaultHttpOptionsFromConfig(config)); + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index c3e3d52a72..65654efd70 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -35,7 +35,7 @@ import { import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; -import { map, startWith } from 'rxjs/operators'; +import { map, startWith, switchMap } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; @@ -136,10 +136,22 @@ export class CalculatedFieldDialogComponent extends DialogComponent { - this.configFormGroup.get('expressionSCRIPT').setValue(expression); - this.configFormGroup.get('expressionSCRIPT').markAsDirty(); - }); + const calculatedFieldId = this.data.value?.id?.id; + + (calculatedFieldId + ? this.calculatedFieldsService.getLatestCalculatedFieldDebugEvent(calculatedFieldId) + .pipe( + switchMap(event => { + const args = event?.arguments ? JSON.parse(event.arguments) : null; + return this.data.getTestScriptDialogFn(this.fromGroupValue, args, false); + }), + takeUntilDestroyed(this.destroyRef) + ) + : this.data.getTestScriptDialogFn(this.fromGroupValue, null, false)) + .subscribe(expression => { + this.configFormGroup.get('expressionSCRIPT').setValue(expression); + this.configFormGroup.get('expressionSCRIPT').markAsDirty(); + }); } private applyDialogData(): void { From 99beae211577c39944c136e411ab15d9e8b6f0a2 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 28 Feb 2025 17:17:55 +0200 Subject: [PATCH 09/22] refactoring --- .../calculated-field-dialog.component.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 65654efd70..68abd646e3 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -39,6 +39,7 @@ import { map, startWith, switchMap } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { Observable } from 'rxjs'; @Component({ selector: 'tb-calculated-field-dialog', @@ -137,9 +138,10 @@ export class CalculatedFieldDialogComponent extends DialogComponent; - (calculatedFieldId - ? this.calculatedFieldsService.getLatestCalculatedFieldDebugEvent(calculatedFieldId) + if (calculatedFieldId) { + testScriptDialogResult$ = this.calculatedFieldsService.getLatestCalculatedFieldDebugEvent(calculatedFieldId) .pipe( switchMap(event => { const args = event?.arguments ? JSON.parse(event.arguments) : null; @@ -147,11 +149,14 @@ export class CalculatedFieldDialogComponent extends DialogComponent { - this.configFormGroup.get('expressionSCRIPT').setValue(expression); - this.configFormGroup.get('expressionSCRIPT').markAsDirty(); - }); + } else { + testScriptDialogResult$ = this.data.getTestScriptDialogFn(this.fromGroupValue, null, false); + } + + testScriptDialogResult$.subscribe(expression => { + this.configFormGroup.get('expressionSCRIPT').setValue(expression); + this.configFormGroup.get('expressionSCRIPT').markAsDirty(); + }); } private applyDialogData(): void { From ac52098207e4330fa44225f07bc08614d019bf9f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 28 Feb 2025 17:46:25 +0200 Subject: [PATCH 10/22] CF: merge function draft --- .../api/tbel/DefaultTbelInvokeService.java | 1 + .../script/api/tbel/TbTimeWindow.java | 2 + .../thingsboard/script/api/tbel/TbUtils.java | 51 ++++++++++++++-- .../api/tbel/TbelCfTsMultiDoubleVal.java | 32 ++++++++++ .../script/api/tbel/TbelCfTsRollingArg.java | 5 -- .../script/api/tbel/TbelCfTsRollingData.java | 61 +++++++++++++++++++ 6 files changed, 143 insertions(+), 9 deletions(-) create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingData.java diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java index f626746a64..962d384a46 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java @@ -136,6 +136,7 @@ public class DefaultTbelInvokeService extends AbstractScriptInvokeService implem parserConfig.registerDataType("TbelCfSingleValueArg", TbelCfSingleValueArg.class, TbelCfSingleValueArg::memorySize); parserConfig.registerDataType("TbelCfTsRollingArg", TbelCfTsRollingArg.class, TbelCfTsRollingArg::memorySize); parserConfig.registerDataType("TbelCfTsDoubleVal", TbelCfTsDoubleVal.class, TbelCfTsDoubleVal::memorySize); + parserConfig.registerDataType("TbelCfTsRollingData", TbelCfTsRollingData.class, TbelCfTsRollingData::memorySize); parserConfig.registerDataType("TbTimeWindow", TbTimeWindow.class, TbTimeWindow::memorySize); TbUtils.register(parserConfig); executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "tbel-executor")); diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java index ce61965317..5048611838 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java @@ -17,9 +17,11 @@ package org.thingsboard.script.api.tbel; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; @Data @AllArgsConstructor +@NoArgsConstructor public class TbTimeWindow implements TbelCfObject { public static final long OBJ_SIZE = 32L; diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 9a57240d23..b108bbe33e 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2025 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -44,6 +44,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.regex.Matcher; import static java.lang.Character.MAX_RADIX; @@ -367,6 +368,8 @@ public class TbUtils { byte[].class, int.class))); parserConfig.addImport("parseBinaryArrayToInt", new MethodStub(TbUtils.class.getMethod("parseBinaryArrayToInt", byte[].class, int.class, int.class))); + parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("mergeCfTsRollingArgs", + TbelCfTsRollingArg.class, TbelCfTsRollingArg.class))); } public static String btoa(String input) { @@ -1506,5 +1509,45 @@ public class TbUtils { } return hex; } + + public static TbelCfTsRollingData mergeCfTsRollingArgs(TbelCfTsRollingArg a, TbelCfTsRollingArg b) { + return mergeCfTsRollingArgs(Arrays.asList(a, b), null); + } + + public static TbelCfTsRollingData mergeCfTsRollingArgs(List args, Map settings) { + TreeSet allTimestamps = new TreeSet<>(); + long startTs = Long.MAX_VALUE; + long endTs = Long.MIN_VALUE; + for (TbelCfTsRollingArg arg : args) { + for (TbelCfTsDoubleVal val : arg.getValues()) { + allTimestamps.add(val.getTs()); + } + startTs = Math.min(startTs, arg.getTimeWindow().getStartTs()); + endTs = Math.max(endTs, arg.getTimeWindow().getEndTs()); + } + + List data = new ArrayList<>(); + + int[] lastIndex = new int[args.size()]; + double[] result = new double[args.size()]; + Arrays.fill(result, Double.NaN); + + var tw = new TbTimeWindow(startTs, endTs, allTimestamps.size()); + + for (long ts : allTimestamps) { + for (int i = 0; i < args.size(); i++) { + var arg = args.get(i); + var values = arg.getValues(); + while (lastIndex[i] < values.size() && values.get(lastIndex[i]).getTs() <= ts) { + result[i] = values.get(lastIndex[i]).getValue(); + lastIndex[i]++; + } + } + data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); + } + + return new TbelCfTsRollingData(tw, data); + } + } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java new file mode 100644 index 0000000000..33e4a65bb8 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import lombok.Data; + +@Data +public class TbelCfTsMultiDoubleVal implements TbelCfObject { + + public static final long OBJ_SIZE = 32L; // Approximate calculation; + + private final long ts; + private final double[] values; + + @Override + public long memorySize() { + return OBJ_SIZE + values.length * 8L; + } +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java index 807d498a16..74c9d77454 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java @@ -266,11 +266,6 @@ public class TbelCfTsRollingArg implements TbelCfArg, Iterable action) { - values.forEach(action); - } - @Override public String getType() { return "TS_ROLLING"; diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingData.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingData.java new file mode 100644 index 0000000000..646e826915 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingData.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.function.Consumer; + +import static org.thingsboard.script.api.tbel.TbelCfTsDoubleVal.OBJ_SIZE; + +public class TbelCfTsRollingData implements TbelCfObject, Iterable { + + @Getter + private final TbTimeWindow timeWindow; + @Getter + private final List values; + + public TbelCfTsRollingData(TbTimeWindow timeWindow, List values) { + this.timeWindow = timeWindow; + this.values = Collections.unmodifiableList(values); + } + + @Override + public long memorySize() { + return 12 + values.size() * OBJ_SIZE; + } + + @JsonIgnore + public List getValue() { + return values; + } + + @JsonIgnore + public int getSize() { + return values.size(); + } + + @Override + public Iterator iterator() { + return values.iterator(); + } + +} From aecefc84bfdc47afe60b343359d761e5a04f5401 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 3 Mar 2025 19:23:02 +0200 Subject: [PATCH 11/22] fixed NPE, fixed flaky test --- .../subscription/DefaultTbEntityDataSubscriptionService.java | 4 +++- .../org/thingsboard/server/controller/WebsocketApiTest.java | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java index 78d7f01404..9e9ca42e83 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java @@ -436,7 +436,9 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc ctx.sendWsMsg(update); } else { ctx.doFetchAlarmCount(); - ctx.createAlarmSubscriptions(); + if (entitiesIds != null) { + ctx.createAlarmSubscriptions(); + } TbAlarmCountSubCtx finalCtx = ctx; ScheduledFuture task = scheduler.scheduleWithFixedDelay( () -> refreshDynamicQuery(finalCtx), diff --git a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java index 69d50c61d1..9801907d3b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java @@ -83,7 +83,8 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @Slf4j @DaoSqlTest @TestPropertySource(properties = { - "server.ws.alarms_per_alarm_status_subscription_cache_size=5" + "server.ws.alarms_per_alarm_status_subscription_cache_size=5", + "server.ws.dynamic_page_link.refresh_interval=15" }) public class WebsocketApiTest extends AbstractControllerTest { @Autowired From eadc70581725b9f234e89dedda5d3d06e0020233 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 4 Mar 2025 09:16:24 +0200 Subject: [PATCH 12/22] added upgrade script for tenant profile --- .../main/data/upgrade/basic/schema_update.sql | 32 ++++++++++++++++++- .../DefaultTenantProfileConfiguration.java | 10 +++--- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index ddfac0d768..d694a2bcf6 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -63,4 +63,34 @@ $$; -- UPDATE SAVE TIME SERIES NODES END -ALTER TABLE api_usage_state ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; \ No newline at end of file +ALTER TABLE api_usage_state ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; + +-- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS START + +DO $$ + BEGIN + -- Check if the tenant_profile table exists + IF EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_name = 'tenant_profile' + ) THEN + + UPDATE tenant_profile + SET profile_data = profile_data + || jsonb_build_object( + 'configuration', profile_data->'configuration' || jsonb_build_object( + 'maxCalculatedFieldsPerEntity', COALESCE(profile_data->'configuration'->>'maxCalculatedFieldsPerEntity', '5')::bigint, + 'maxArgumentsPerCF', COALESCE(profile_data->'configuration'->>'maxArgumentsPerCF', '10')::bigint, + 'maxDataPointsPerRollingArg', COALESCE(profile_data->'configuration'->>'maxDataPointsPerRollingArg', '1000')::bigint, + 'maxStateSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxStateSizeInKBytes', '32')::bigint, + 'maxSingleValueArgumentSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxSingleValueArgumentSizeInKBytes', '2')::bigint + ) + ) + WHERE profile_data->'configuration' IS NOT NULL; + +END IF; + END; +$$; + +-- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS END diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index 9be8a126b2..9c86785a5e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -135,11 +135,11 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private double warnThreshold; - private long maxCalculatedFieldsPerEntity; - private long maxArgumentsPerCF; - private long maxDataPointsPerRollingArg; - private long maxStateSizeInKBytes; - private long maxSingleValueArgumentSizeInKBytes; + private long maxCalculatedFieldsPerEntity = 5; + private long maxArgumentsPerCF = 10; + private long maxDataPointsPerRollingArg = 1000; + private long maxStateSizeInKBytes = 32; + private long maxSingleValueArgumentSizeInKBytes = 2; @Override public long getProfileThreshold(ApiUsageRecordKey key) { From 7994802e4bc3390c6c6701ff726289072ce2890c Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 4 Mar 2025 09:39:31 +0200 Subject: [PATCH 13/22] added default values on the UI --- ui-ngx/src/app/shared/models/tenant.model.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 38ff63e3c6..23896141db 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -156,11 +156,11 @@ export function createTenantProfileConfiguration(type: TenantProfileType): Tenan rpcTtlDays: 0, queueStatsTtlDays: 0, ruleEngineExceptionsTtlDays: 0, - maxCalculatedFieldsPerEntity: 0, - maxArgumentsPerCF: 0, - maxDataPointsPerRollingArg: 0, - maxStateSizeInKBytes: 0, - maxSingleValueArgumentSizeInKBytes: 0, + maxCalculatedFieldsPerEntity: 5, + maxArgumentsPerCF: 10, + maxDataPointsPerRollingArg: 1000, + maxStateSizeInKBytes: 32, + maxSingleValueArgumentSizeInKBytes: 2, calculatedFieldDebugEventsRateLimit: '' }; configuration = {...defaultConfiguration, type: TenantProfileType.DEFAULT}; From afd727b097ae4253c7bd8fe445a0c9197eb2d4ad Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 4 Mar 2025 11:22:51 +0200 Subject: [PATCH 14/22] removed check for changing value when update state and moved timeout to config param --- .../server/actors/ActorSystemContext.java | 4 ++++ ...CalculatedFieldEntityMessageProcessor.java | 21 ++++++++----------- .../ctx/state/SingleValueArgumentEntry.java | 4 ---- .../src/main/resources/thingsboard.yml | 2 ++ 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 50a777840c..9ab7adfcc6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -644,6 +644,10 @@ public class ActorSystemContext { @Getter private String deviceStateNodeRateLimitConfig; + @Value("${actors.calculated_fields.calculation_result_timeout:5}") + @Getter + private long cfCalculationResultTimeout; + @Getter @Setter private TbActorSystem actorSystem; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index f7fc204c0f..279628c566 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -275,9 +275,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); boolean stateSizeOk; - if (ctx.isInitialized() && state.isReady()) { - try { - CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); + try { + if (ctx.isInitialized() && state.isReady()) { + CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(systemContext.getCfCalculationResultTimeout(), TimeUnit.SECONDS); state.checkStateSize(ctxId, ctx.getMaxStateSize()); stateSizeOk = state.isSizeOk(); if (stateSizeOk) { @@ -286,21 +286,18 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); } } - } catch (Exception e) { - throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build(); } - } else { + } catch (Exception e) { + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build(); + } finally { state.checkStateSize(ctxId, ctx.getMaxStateSize()); stateSizeOk = state.isSizeOk(); if (stateSizeOk) { - callback.onSuccess(); // State was updated but no calculation performed; + cfStateService.persistState(ctxId, state, callback); + } else { + removeStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback); } } - if (stateSizeOk) { - cfStateService.persistState(ctxId, state, callback); - } else { - removeStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback); - } } private void removeStateAndRaiseSizeException(CalculatedFieldEntityCtxId ctxId, CalculatedFieldException ex, TbCallback callback) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index a064a99935..a237f3d022 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -100,10 +100,6 @@ public class SingleValueArgumentEntry implements ArgumentEntry { if (newVersion == null || this.version == null || newVersion > this.version) { this.ts = singleValueEntry.getTs(); this.version = newVersion; - BasicKvEntry newValue = singleValueEntry.getKvEntryValue(); - if (this.kvEntryValue != null && this.kvEntryValue.getValue().equals(newValue.getValue())) { - return false; - } this.kvEntryValue = singleValueEntry.getKvEntryValue(); return true; } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index ec9925bf06..d0ccef8ba9 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -512,6 +512,8 @@ actors: enabled: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}" # The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" + # Time in seconds to receive calculation result. + calculation_result_timeout: "${ACTORS_CALCULATION_RESULT_TIMEOUT_SEC:5}" debug: settings: From 8b30d1d93be0b4b5743ff9a77b05395cafc2e661 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 4 Mar 2025 11:31:13 +0200 Subject: [PATCH 15/22] fixed test for new logic --- .../service/cf/ctx/state/SingleValueArgumentEntryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java index e83e30663d..2c48ed9167 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -71,6 +71,6 @@ public class SingleValueArgumentEntryTest { @Test void testUpdateEntryWhenValueWasNotChanged() { - assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 11L), 237L))).isFalse(); + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 11L), 364L))).isTrue(); } } \ No newline at end of file From 6265ff45a8adca7c139d226d81054d287ca406bf Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 4 Mar 2025 11:54:28 +0200 Subject: [PATCH 16/22] merge function draft --- .../api/tbel/DefaultTbelInvokeService.java | 2 + .../script/api/tbel/TbTimeWindow.java | 3 + .../thingsboard/script/api/tbel/TbUtils.java | 60 +++++++++++++--- .../api/tbel/TbelCfTsMultiDoubleVal.java | 34 +++++++++ .../script/api/tbel/TbUtilsTest.java | 72 ++++++++++++++++++- 5 files changed, 160 insertions(+), 11 deletions(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java index 962d384a46..38fed7a2aa 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java @@ -138,6 +138,8 @@ public class DefaultTbelInvokeService extends AbstractScriptInvokeService implem parserConfig.registerDataType("TbelCfTsDoubleVal", TbelCfTsDoubleVal.class, TbelCfTsDoubleVal::memorySize); parserConfig.registerDataType("TbelCfTsRollingData", TbelCfTsRollingData.class, TbelCfTsRollingData::memorySize); parserConfig.registerDataType("TbTimeWindow", TbTimeWindow.class, TbTimeWindow::memorySize); + parserConfig.registerDataType("TbelCfTsDoubleVal", TbelCfTsMultiDoubleVal.class, TbelCfTsMultiDoubleVal::memorySize); + TbUtils.register(parserConfig); executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "tbel-executor")); try { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java index 5048611838..b430b23757 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java @@ -35,4 +35,7 @@ public class TbTimeWindow implements TbelCfObject { return OBJ_SIZE; } + public boolean matches(long ts) { + return ts >= startTs && ts < endTs; + } } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 4796cf12bb..e6a0b8d043 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2025 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -368,8 +368,16 @@ public class TbUtils { byte[].class, int.class))); parserConfig.addImport("parseBinaryArrayToInt", new MethodStub(TbUtils.class.getMethod("parseBinaryArrayToInt", byte[].class, int.class, int.class))); - parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("mergeCfTsRollingArgs", + parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", TbelCfTsRollingArg.class, TbelCfTsRollingArg.class))); + parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", + TbelCfTsRollingArg.class, TbelCfTsRollingArg.class, TbTimeWindow.class))); + parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", + TbelCfTsRollingArg.class, TbelCfTsRollingArg.class, TbTimeWindow.class, Map.class))); + parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", + List.class, TbTimeWindow.class))); + parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", + List.class, TbTimeWindow.class, Map.class))); } public static String btoa(String input) { @@ -1510,11 +1518,28 @@ public class TbUtils { return hex; } - public static TbelCfTsRollingData mergeCfTsRollingArgs(TbelCfTsRollingArg a, TbelCfTsRollingArg b) { - return mergeCfTsRollingArgs(Arrays.asList(a, b), null); + public static TbelCfTsRollingData merge(TbelCfTsRollingArg a, TbelCfTsRollingArg b) { + return merge(Arrays.asList(a, b), null, null); } - public static TbelCfTsRollingData mergeCfTsRollingArgs(List args, Map settings) { + public static TbelCfTsRollingData merge(TbelCfTsRollingArg a, TbelCfTsRollingArg b, TbTimeWindow timeWindow) { + return merge(Arrays.asList(a, b), timeWindow, null); + } + + public static TbelCfTsRollingData merge(TbelCfTsRollingArg a, TbelCfTsRollingArg b, TbTimeWindow timeWindow, Map settings) { + return merge(Arrays.asList(a, b), timeWindow, settings); + } + + public static TbelCfTsRollingData merge(List args, TbTimeWindow timeWindow) { + return merge(args, timeWindow, null); + } + + public static TbelCfTsRollingData merge(List args, TbTimeWindow timeWindow, Map settings) { + boolean ignoreNaN = true; + if (settings != null && settings.containsKey("ignoreNaN")) { + ignoreNaN = Boolean.parseBoolean(settings.get("ignoreNaN").toString()); + } + TreeSet allTimestamps = new TreeSet<>(); long startTs = Long.MAX_VALUE; long endTs = Long.MIN_VALUE; @@ -1532,7 +1557,7 @@ public class TbUtils { double[] result = new double[args.size()]; Arrays.fill(result, Double.NaN); - var tw = new TbTimeWindow(startTs, endTs, allTimestamps.size()); + var tw = timeWindow != null ? timeWindow : new TbTimeWindow(startTs, endTs, allTimestamps.size()); for (long ts : allTimestamps) { for (int i = 0; i < args.size(); i++) { @@ -1543,7 +1568,22 @@ public class TbUtils { lastIndex[i]++; } } - data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); + if (tw.matches(ts)) { + if (ignoreNaN) { + boolean skip = false; + for (int i = 0; i < args.size(); i++) { + if (Double.isNaN(result[i])) { + skip = true; + break; + } + } + if (!skip) { + data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); + } + } else { + data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); + } + } } return new TbelCfTsRollingData(tw, data); diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java index 33e4a65bb8..2743bbd0de 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java @@ -15,6 +15,7 @@ */ package org.thingsboard.script.api.tbel; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; @Data @@ -25,6 +26,39 @@ public class TbelCfTsMultiDoubleVal implements TbelCfObject { private final long ts; private final double[] values; + @JsonIgnore + public double getV1() { + return getV(0); + } + + @JsonIgnore + public double getV2() { + return getV(1); + } + + @JsonIgnore + public double getV3() { + return getV(2); + } + + @JsonIgnore + public double getV4() { + return getV(3); + } + + @JsonIgnore + public double getV5() { + return getV(4); + } + + private double getV(int idx) { + if (values.length < idx + 1) { + throw new IllegalArgumentException("Can't get value at index " + idx + ". There are " + values.length + " values present."); + } else { + return values[idx]; + } + } + @Override public long memorySize() { return OBJ_SIZE + values.length * 8L; diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index fbd81948d6..f19a323068 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -1109,7 +1109,7 @@ public class TbUtilsTest { String validInput = Base64.getEncoder().encodeToString(new byte[]{1, 2, 3, 4, 5}); ExecutionArrayList actual = TbUtils.base64ToBytesList(ctx, validInput); ExecutionArrayList expected = new ExecutionArrayList<>(ctx); - expected.addAll(List.of((byte) 1, (byte)2, (byte)3, (byte)4, (byte)5)); + expected.addAll(List.of((byte) 1, (byte) 2, (byte) 3, (byte) 4, (byte) 5)); Assertions.assertEquals(expected, actual); String emptyInput = Base64.getEncoder().encodeToString(new byte[]{}); @@ -1123,6 +1123,7 @@ public class TbUtilsTest { TbUtils.base64ToBytesList(ctx, null); }); } + @Test public void bytesToHex_Test() { byte[] bb = {(byte) 0xBB, (byte) 0xAA}; @@ -1136,6 +1137,75 @@ public class TbUtilsTest { Assertions.assertEquals(expected, actual); } + @Test + public void merge_two_rolling_args_ts_match_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000, 1000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = TbUtils.merge(arg1, arg2); + Assertions.assertEquals(3, result.getSize()); + Assertions.assertNotNull(result.getValues()); + Assertions.assertNotNull(result.getValues().get(0)); + Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); + Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); + Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); + } + + @Test + public void merge_two_rolling_args_with_timewindow_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000, 1000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = TbUtils.merge(arg1, arg2, new TbTimeWindow(0, 10000, 1000)); + Assertions.assertEquals(2, result.getSize()); + Assertions.assertNotNull(result.getValues()); + Assertions.assertNotNull(result.getValues().get(0)); + Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); + Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); + Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); + } + + @Test + public void merge_two_rolling_args_ts_mismatch_default_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000, 1000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(100, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(200, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = TbUtils.merge(arg1, arg2); + Assertions.assertEquals(3, result.getSize()); + Assertions.assertNotNull(result.getValues()); + + TbelCfTsMultiDoubleVal item0 = result.getValues().get(0); + Assertions.assertNotNull(item0); + Assertions.assertEquals(200L, item0.getTs()); + Assertions.assertEquals(1, item0.getValues()[0]); + Assertions.assertEquals(11, item0.getValues()[1]); + } + + @Test + public void merge_two_rolling_args_ts_mismatch_ignore_nan_disabled_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000, 1000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(100, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(200, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = TbUtils.merge(Arrays.asList(arg1, arg2), new TbTimeWindow(0, 60000, 1000), Collections.singletonMap("ignoreNaN", false)); + Assertions.assertEquals(4, result.getSize()); + Assertions.assertNotNull(result.getValues()); + + TbelCfTsMultiDoubleVal item0 = result.getValues().get(0); + Assertions.assertNotNull(item0); + Assertions.assertEquals(100L, item0.getTs()); + Assertions.assertEquals(1, item0.getValues()[0]); + Assertions.assertEquals(Double.NaN, item0.getValues()[1]); + + TbelCfTsMultiDoubleVal item1 = result.getValues().get(1); + Assertions.assertEquals(200L, item1.getTs()); + Assertions.assertEquals(1, item1.getValues()[0]); + Assertions.assertEquals(11, item1.getValues()[1]); + } + private static List toList(byte[] data) { List result = new ArrayList<>(data.length); for (Byte b : data) { From 759bc94d1d797f521c3326ad777c4b45834197ff Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 4 Mar 2025 11:57:57 +0200 Subject: [PATCH 17/22] Review comments --- .../server/actors/ActorSystemContext.java | 2 +- .../CalculatedFieldEntityMessageProcessor.java | 13 +++++++------ application/src/main/resources/thingsboard.yml | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 9ab7adfcc6..de3864df9f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -644,7 +644,7 @@ public class ActorSystemContext { @Getter private String deviceStateNodeRateLimitConfig; - @Value("${actors.calculated_fields.calculation_result_timeout:5}") + @Value("${actors.calculated_fields.calculation_timeout:5}") @Getter private long cfCalculationResultTimeout; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 279628c566..e42ebb593e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -274,13 +274,13 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); - boolean stateSizeOk; + boolean stateSizeChecked = false; try { if (ctx.isInitialized() && state.isReady()) { CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(systemContext.getCfCalculationResultTimeout(), TimeUnit.SECONDS); state.checkStateSize(ctxId, ctx.getMaxStateSize()); - stateSizeOk = state.isSizeOk(); - if (stateSizeOk) { + stateSizeChecked = true; + if (state.isSizeOk()) { cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); @@ -290,9 +290,10 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } catch (Exception e) { throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build(); } finally { - state.checkStateSize(ctxId, ctx.getMaxStateSize()); - stateSizeOk = state.isSizeOk(); - if (stateSizeOk) { + if (!stateSizeChecked) { + state.checkStateSize(ctxId, ctx.getMaxStateSize()); + } + if (state.isSizeOk()) { cfStateService.persistState(ctxId, state, callback); } else { removeStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index d0ccef8ba9..53258a4296 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -513,7 +513,7 @@ actors: # The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" # Time in seconds to receive calculation result. - calculation_result_timeout: "${ACTORS_CALCULATION_RESULT_TIMEOUT_SEC:5}" + calculation_timeout: "${ACTORS_CALCULATION_TIMEOUT_SEC:5}" debug: settings: From 933d77427ce0eb12967d24f94be279e26fbd5eed Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 4 Mar 2025 13:13:09 +0200 Subject: [PATCH 18/22] Merge function --- .../cf/ctx/state/TsRollingArgumentEntry.java | 2 +- .../script/api/tbel/TbTimeWindow.java | 1 - .../thingsboard/script/api/tbel/TbUtils.java | 81 --------------- .../script/api/tbel/TbelCfTsRollingArg.java | 98 ++++++++++++++++++- .../script/api/tbel/TbUtilsTest.java | 69 ------------- .../api/tbel/TbelCfTsRollingArgTest.java | 88 ++++++++++++++++- .../shared/models/calculated-field.models.ts | 89 +++++++++++++---- 7 files changed, 254 insertions(+), 174 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 1114d993a3..b5a680a072 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -89,7 +89,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { for (var e : tsRecords.entrySet()) { values.add(new TbelCfTsDoubleVal(e.getKey(), e.getValue())); } - return new TbelCfTsRollingArg(limit, timeWindow, values); + return new TbelCfTsRollingArg(timeWindow, values); } @Override diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java index b430b23757..1761ef6d0a 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java @@ -28,7 +28,6 @@ public class TbTimeWindow implements TbelCfObject { private long startTs; private long endTs; - private int limit; @Override public long memorySize() { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index e6a0b8d043..a43cc30f23 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -368,16 +368,6 @@ public class TbUtils { byte[].class, int.class))); parserConfig.addImport("parseBinaryArrayToInt", new MethodStub(TbUtils.class.getMethod("parseBinaryArrayToInt", byte[].class, int.class, int.class))); - parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", - TbelCfTsRollingArg.class, TbelCfTsRollingArg.class))); - parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", - TbelCfTsRollingArg.class, TbelCfTsRollingArg.class, TbTimeWindow.class))); - parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", - TbelCfTsRollingArg.class, TbelCfTsRollingArg.class, TbTimeWindow.class, Map.class))); - parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", - List.class, TbTimeWindow.class))); - parserConfig.addImport("merge", new MethodStub(TbUtils.class.getMethod("merge", - List.class, TbTimeWindow.class, Map.class))); } public static String btoa(String input) { @@ -1518,76 +1508,5 @@ public class TbUtils { return hex; } - public static TbelCfTsRollingData merge(TbelCfTsRollingArg a, TbelCfTsRollingArg b) { - return merge(Arrays.asList(a, b), null, null); - } - - public static TbelCfTsRollingData merge(TbelCfTsRollingArg a, TbelCfTsRollingArg b, TbTimeWindow timeWindow) { - return merge(Arrays.asList(a, b), timeWindow, null); - } - - public static TbelCfTsRollingData merge(TbelCfTsRollingArg a, TbelCfTsRollingArg b, TbTimeWindow timeWindow, Map settings) { - return merge(Arrays.asList(a, b), timeWindow, settings); - } - - public static TbelCfTsRollingData merge(List args, TbTimeWindow timeWindow) { - return merge(args, timeWindow, null); - } - - public static TbelCfTsRollingData merge(List args, TbTimeWindow timeWindow, Map settings) { - boolean ignoreNaN = true; - if (settings != null && settings.containsKey("ignoreNaN")) { - ignoreNaN = Boolean.parseBoolean(settings.get("ignoreNaN").toString()); - } - - TreeSet allTimestamps = new TreeSet<>(); - long startTs = Long.MAX_VALUE; - long endTs = Long.MIN_VALUE; - for (TbelCfTsRollingArg arg : args) { - for (TbelCfTsDoubleVal val : arg.getValues()) { - allTimestamps.add(val.getTs()); - } - startTs = Math.min(startTs, arg.getTimeWindow().getStartTs()); - endTs = Math.max(endTs, arg.getTimeWindow().getEndTs()); - } - - List data = new ArrayList<>(); - - int[] lastIndex = new int[args.size()]; - double[] result = new double[args.size()]; - Arrays.fill(result, Double.NaN); - - var tw = timeWindow != null ? timeWindow : new TbTimeWindow(startTs, endTs, allTimestamps.size()); - - for (long ts : allTimestamps) { - for (int i = 0; i < args.size(); i++) { - var arg = args.get(i); - var values = arg.getValues(); - while (lastIndex[i] < values.size() && values.get(lastIndex[i]).getTs() <= ts) { - result[i] = values.get(lastIndex[i]).getValue(); - lastIndex[i]++; - } - } - if (tw.matches(ts)) { - if (ignoreNaN) { - boolean skip = false; - for (int i = 0; i < args.size(); i++) { - if (Double.isNaN(result[i])) { - skip = true; - break; - } - } - if (!skip) { - data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); - } - } else { - data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); - } - } - } - - return new TbelCfTsRollingData(tw, data); - } - } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java index 74c9d77454..3b5a7ac9bd 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java @@ -19,11 +19,15 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Getter; +import org.thingsboard.common.util.JacksonUtil; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.TreeSet; import java.util.function.Consumer; import static org.thingsboard.script.api.tbel.TbelCfTsDoubleVal.OBJ_SIZE; @@ -44,9 +48,9 @@ public class TbelCfTsRollingArg implements TbelCfArg, Iterable values) { + public TbelCfTsRollingArg(long timeWindow, List values) { long ts = System.currentTimeMillis(); - this.timeWindow = new TbTimeWindow(ts - timeWindow, ts, limit); + this.timeWindow = new TbTimeWindow(ts - timeWindow, ts); this.values = Collections.unmodifiableList(values); } @@ -104,6 +108,14 @@ public class TbelCfTsRollingArg implements TbelCfArg, Iterable settings) { + return mergeAll(Collections.singletonList(other), settings); + } + + public TbelCfTsRollingData mergeAll(List others) { + return mergeAll(others, null); + } + + public TbelCfTsRollingData mergeAll(List others, Map settings) { + List args = new ArrayList<>(others.size() + 1); + args.add(this); + args.addAll(others); + + boolean ignoreNaN = true; + if (settings != null && settings.containsKey("ignoreNaN")) { + ignoreNaN = Boolean.parseBoolean(settings.get("ignoreNaN").toString()); + } + + TbTimeWindow timeWindow = null; + if (settings != null && settings.containsKey("timeWindow")) { + var twVar = settings.get("timeWindow"); + if (twVar instanceof TbTimeWindow) { + timeWindow = (TbTimeWindow) settings.get("timeWindow"); + } else if (twVar instanceof Map twMap) { + timeWindow = new TbTimeWindow(Long.valueOf(twMap.get("startTs").toString()), Long.valueOf(twMap.get("endTs").toString())); + } else { + timeWindow = JacksonUtil.fromString(settings.get("timeWindow").toString(), TbTimeWindow.class); + } + } + + TreeSet allTimestamps = new TreeSet<>(); + long startTs = Long.MAX_VALUE; + long endTs = Long.MIN_VALUE; + for (TbelCfTsRollingArg arg : args) { + for (TbelCfTsDoubleVal val : arg.getValues()) { + allTimestamps.add(val.getTs()); + } + startTs = Math.min(startTs, arg.getTimeWindow().getStartTs()); + endTs = Math.max(endTs, arg.getTimeWindow().getEndTs()); + } + + List data = new ArrayList<>(); + + int[] lastIndex = new int[args.size()]; + double[] result = new double[args.size()]; + Arrays.fill(result, Double.NaN); + + for (long ts : allTimestamps) { + for (int i = 0; i < args.size(); i++) { + var arg = args.get(i); + var values = arg.getValues(); + while (lastIndex[i] < values.size() && values.get(lastIndex[i]).getTs() <= ts) { + result[i] = values.get(lastIndex[i]).getValue(); + lastIndex[i]++; + } + } + if (timeWindow == null || timeWindow.matches(ts)) { + if (ignoreNaN) { + boolean skip = false; + for (int i = 0; i < args.size(); i++) { + if (Double.isNaN(result[i])) { + skip = true; + break; + } + } + if (!skip) { + data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); + } + } else { + data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); + } + } + } + + return new TbelCfTsRollingData(timeWindow != null ? timeWindow : new TbTimeWindow(startTs, endTs), data); + } + + @JsonIgnore public int getSize() { return values.size(); diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index f19a323068..71c7a8b78a 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -1137,75 +1137,6 @@ public class TbUtilsTest { Assertions.assertEquals(expected, actual); } - @Test - public void merge_two_rolling_args_ts_match_test() { - TbTimeWindow tw = new TbTimeWindow(0, 60000, 1000); - TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); - TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); - - var result = TbUtils.merge(arg1, arg2); - Assertions.assertEquals(3, result.getSize()); - Assertions.assertNotNull(result.getValues()); - Assertions.assertNotNull(result.getValues().get(0)); - Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); - Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); - Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); - } - - @Test - public void merge_two_rolling_args_with_timewindow_test() { - TbTimeWindow tw = new TbTimeWindow(0, 60000, 1000); - TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); - TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); - - var result = TbUtils.merge(arg1, arg2, new TbTimeWindow(0, 10000, 1000)); - Assertions.assertEquals(2, result.getSize()); - Assertions.assertNotNull(result.getValues()); - Assertions.assertNotNull(result.getValues().get(0)); - Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); - Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); - Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); - } - - @Test - public void merge_two_rolling_args_ts_mismatch_default_test() { - TbTimeWindow tw = new TbTimeWindow(0, 60000, 1000); - TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(100, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); - TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(200, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); - - var result = TbUtils.merge(arg1, arg2); - Assertions.assertEquals(3, result.getSize()); - Assertions.assertNotNull(result.getValues()); - - TbelCfTsMultiDoubleVal item0 = result.getValues().get(0); - Assertions.assertNotNull(item0); - Assertions.assertEquals(200L, item0.getTs()); - Assertions.assertEquals(1, item0.getValues()[0]); - Assertions.assertEquals(11, item0.getValues()[1]); - } - - @Test - public void merge_two_rolling_args_ts_mismatch_ignore_nan_disabled_test() { - TbTimeWindow tw = new TbTimeWindow(0, 60000, 1000); - TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(100, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); - TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(200, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); - - var result = TbUtils.merge(Arrays.asList(arg1, arg2), new TbTimeWindow(0, 60000, 1000), Collections.singletonMap("ignoreNaN", false)); - Assertions.assertEquals(4, result.getSize()); - Assertions.assertNotNull(result.getValues()); - - TbelCfTsMultiDoubleVal item0 = result.getValues().get(0); - Assertions.assertNotNull(item0); - Assertions.assertEquals(100L, item0.getTs()); - Assertions.assertEquals(1, item0.getValues()[0]); - Assertions.assertEquals(Double.NaN, item0.getValues()[1]); - - TbelCfTsMultiDoubleVal item1 = result.getValues().get(1); - Assertions.assertEquals(200L, item1.getTs()); - Assertions.assertEquals(1, item1.getValues()[0]); - Assertions.assertEquals(11, item1.getValues()[1]); - } - private static List toList(byte[] data) { List result = new ArrayList<>(data.length); for (Byte b : data) { diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java index 327b477b6f..69eba2fab2 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java @@ -15,10 +15,15 @@ */ package org.thingsboard.script.api.tbel; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.thingsboard.common.util.JacksonUtil; +import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -33,7 +38,7 @@ public class TbelCfTsRollingArgTest { @BeforeEach void setUp() { rollingArg = new TbelCfTsRollingArg( - new TbTimeWindow(ts - 30000, ts - 10, 10), + new TbTimeWindow(ts - 30000, ts - 10), List.of( new TbelCfTsDoubleVal(ts - 10, Double.NaN), new TbelCfTsDoubleVal(ts - 20, 2.0), @@ -98,7 +103,7 @@ public class TbelCfTsRollingArgTest { void testFirstAndLastWhenOnlyNaNAndIgnoreNaNIsFalse() { assertThat(rollingArg.first()).isEqualTo(2.0); rollingArg = new TbelCfTsRollingArg( - new TbTimeWindow(ts - 30000, ts - 10, 10), + new TbTimeWindow(ts - 30000, ts - 10), List.of( new TbelCfTsDoubleVal(ts - 10, Double.NaN), new TbelCfTsDoubleVal(ts - 40, Double.NaN), @@ -117,7 +122,7 @@ public class TbelCfTsRollingArgTest { @Test void testEmptyValues() { - rollingArg = new TbelCfTsRollingArg(new TbTimeWindow(0, 10, 10), List.of()); + rollingArg = new TbelCfTsRollingArg(new TbTimeWindow(0, 10), List.of()); assertThatThrownBy(rollingArg::sum).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); assertThatThrownBy(rollingArg::max).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); assertThatThrownBy(rollingArg::min).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); @@ -128,4 +133,81 @@ public class TbelCfTsRollingArgTest { assertThatThrownBy(rollingArg::last).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); } + @Test + public void merge_two_rolling_args_ts_match_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = arg1.merge(arg2); + Assertions.assertEquals(3, result.getSize()); + Assertions.assertNotNull(result.getValues()); + Assertions.assertNotNull(result.getValues().get(0)); + Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); + Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); + Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); + } + + @Test + public void merge_two_rolling_args_with_timewindow_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = arg1.merge(arg2, Collections.singletonMap("timeWindow", new TbTimeWindow(0, 10000))); + Assertions.assertEquals(2, result.getSize()); + Assertions.assertNotNull(result.getValues()); + Assertions.assertNotNull(result.getValues().get(0)); + Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); + Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); + Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); + + result = arg1.merge(arg2, Collections.singletonMap("timeWindow", Map.of("startTs", 0L, "endTs", 10000))); + Assertions.assertEquals(2, result.getSize()); + Assertions.assertNotNull(result.getValues()); + Assertions.assertNotNull(result.getValues().get(0)); + Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); + Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); + Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); + } + + @Test + public void merge_two_rolling_args_ts_mismatch_default_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(100, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(200, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = arg1.merge(arg2); + Assertions.assertEquals(3, result.getSize()); + Assertions.assertNotNull(result.getValues()); + + TbelCfTsMultiDoubleVal item0 = result.getValues().get(0); + Assertions.assertNotNull(item0); + Assertions.assertEquals(200L, item0.getTs()); + Assertions.assertEquals(1, item0.getValues()[0]); + Assertions.assertEquals(11, item0.getValues()[1]); + } + + @Test + public void merge_two_rolling_args_ts_mismatch_ignore_nan_disabled_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(100, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(200, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = arg1.merge(arg2, Collections.singletonMap("ignoreNaN", false)); + Assertions.assertEquals(4, result.getSize()); + Assertions.assertNotNull(result.getValues()); + + TbelCfTsMultiDoubleVal item0 = result.getValues().get(0); + Assertions.assertNotNull(item0); + Assertions.assertEquals(100L, item0.getTs()); + Assertions.assertEquals(1, item0.getValues()[0]); + Assertions.assertEquals(Double.NaN, item0.getValues()[1]); + + TbelCfTsMultiDoubleVal item1 = result.getValues().get(1); + Assertions.assertEquals(200L, item1.getTs()); + Assertions.assertEquals(1, item1.getValues()[0]); + Assertions.assertEquals(11, item1.getValues()[1]); + } + } \ No newline at end of file diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index f6d7aaaa1e..4467fecc79 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -272,7 +272,7 @@ export const CalculatedFieldAttributeValueArgumentAutocomplete = { export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { max: { meta: 'function', - description: 'Computes the maximum value in the list of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + description: 'Returns the maximum value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', args: [ { name: 'ignoreNaN', @@ -288,7 +288,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, min: { meta: 'function', - description: 'Computes the minimum value in the list of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + description: 'Returns the minimum value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', args: [ { name: 'ignoreNaN', @@ -304,7 +304,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, mean: { meta: 'function', - description: 'Computes the mean value of the rolling argument values list. Returns NaN if any value is NaN and ignoreNaN is false.', + description: 'Computes the mean value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', args: [ { name: 'ignoreNaN', @@ -318,9 +318,25 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { type: 'number' } }, + avg: { + meta: 'function', + description: 'Computes the average value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The average value, or NaN if applicable', + type: 'number' + } + }, std: { meta: 'function', - description: 'Computes the standard deviation in the list of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + description: 'Computes the standard deviation of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', args: [ { name: 'ignoreNaN', @@ -336,7 +352,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, median: { meta: 'function', - description: 'Computes the median value of the rolling argument values list. Returns NaN if any value is NaN and ignoreNaN is false.', + description: 'Computes the median value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', args: [ { name: 'ignoreNaN', @@ -352,7 +368,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, count: { meta: 'function', - description: 'Counts values in the list of rolling argument values. Counts non-NaN values if ignoreNaN is true, otherwise - total size.', + description: 'Counts values of the rolling argument. Counts non-NaN values if ignoreNaN is true, otherwise - total size.', args: [ { name: 'ignoreNaN', @@ -368,7 +384,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, last: { meta: 'function', - description: 'Returns the last non-NaN value in the list of rolling argument values if ignoreNaN is true, otherwise - the last value.', + description: 'Returns the last non-NaN value of the rolling argument values if ignoreNaN is true, otherwise - the last value.', args: [ { name: 'ignoreNaN', @@ -384,7 +400,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, first: { meta: 'function', - description: 'Returns the first non-NaN value in the list of rolling argument values if ignoreNaN is true, otherwise - the first value.', + description: 'Returns the first non-NaN value of the rolling argument values if ignoreNaN is true, otherwise - the first value.', args: [ { name: 'ignoreNaN', @@ -400,7 +416,7 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { }, sum: { meta: 'function', - description: 'Computes the sum of values in the list of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + description: 'Computes the sum of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', args: [ { name: 'ignoreNaN', @@ -413,12 +429,56 @@ export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { description: 'The sum of values, or NaN if applicable', type: 'number' } + }, + merge: { + meta: 'function', + description: 'Merges current object with other time series rolling argument into a single object by aligning their timestamped values. Supports optional configurable settings.', + args: [ + { + name: 'other', + description: "A time series rolling argument to be merged with the current object.", + type: "object", + optional: true + }, + { + name: "settings", + description: "Optional settings controlling the merging process. Supported keys: 'ignoreNaN' (boolean, equals true by default) to determine whether NaN values should be ignored; 'timeWindow' (object, empty by default) to apply time window filtering.", + type: "object", + optional: true + } + ], + return: { + description: 'A new object containing merged timestamped values from all provided arguments, aligned based on timestamps and filtered according to settings.', + type: '{ values: { ts: number; values: number[]; }[]; timeWindow: { startTs: number; endTs: number } }; }', + } + }, + mergeAll: { + meta: 'function', + description: 'Merges current object with other time series rolling arguments into a single object by aligning their timestamped values. Supports optional configurable settings.', + args: [ + { + name: 'others', + description: "A list of time series rolling arguments to be merged with the current object.", + type: "object[]", + optional: true + }, + { + name: "settings", + description: "Optional settings controlling the merging process. Supported keys: 'ignoreNaN' (boolean, equals true by default) to determine whether NaN values should be ignored; 'timeWindow' (object, empty by default) to apply time window filtering.", + type: "object", + optional: true + } + ], + return: { + description: 'A new object containing merged timestamped values from all provided arguments, aligned based on timestamps and filtered according to settings.', + type: '{ values: { ts: number; values: number[]; }[]; timeWindow: { startTs: number; endTs: number } }; }', + } } }; export const CalculatedFieldRollingValueArgumentAutocomplete = { meta: 'object', - type: '{ values: { ts: number; value: any; }[]; timeWindow: { startTs: number; endTs: number; limit: number } }; }', + type: '{ values: { ts: number; value: number; }[]; timeWindow: { startTs: number; endTs: number } }; }', description: 'Calculated field rolling value argument.', children: { ...CalculatedFieldRollingValueArgumentFunctionsAutocomplete, @@ -429,7 +489,7 @@ export const CalculatedFieldRollingValueArgumentAutocomplete = { }, timeWindow: { meta: 'object', - type: '{ startTs: number; endTs: number; limit: number }', + type: '{ startTs: number; endTs: number }', description: 'Time window configuration', children: { startTs: { @@ -441,11 +501,6 @@ export const CalculatedFieldRollingValueArgumentAutocomplete = { meta: 'number', type: 'number', description: 'End time stamp', - }, - limit: { - meta: 'number', - type: 'number', - description: 'Limit', } } } @@ -504,7 +559,7 @@ const calculatedFieldSingleArgumentValueHighlightRules: AceHighlightRules = { } const calculatedFieldRollingArgumentValueFunctionsHighlightRules: Array = - ['max', 'min', 'mean', 'std', 'median', 'count', 'last', 'first', 'sum'].map(funcName => ({ + ['max', 'min', 'avg', 'mean', 'std', 'median', 'count', 'last', 'first', 'sum', 'merge', 'mergeAll'].map(funcName => ({ token: 'tb.calculated-field-func', regex: `\\b${funcName}\\b`, next: 'no_regex' From 79d0e706d2188a6740f3b0fc83a0fd3df411960d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 4 Mar 2025 14:02:53 +0200 Subject: [PATCH 19/22] updated upgrade script --- .../main/data/upgrade/basic/schema_update.sql | 37 ++++++------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index d694a2bcf6..e9cbf8ef8e 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -67,30 +67,17 @@ ALTER TABLE api_usage_state ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; -- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS START -DO $$ - BEGIN - -- Check if the tenant_profile table exists - IF EXISTS ( - SELECT 1 - FROM information_schema.tables - WHERE table_name = 'tenant_profile' - ) THEN - - UPDATE tenant_profile - SET profile_data = profile_data - || jsonb_build_object( - 'configuration', profile_data->'configuration' || jsonb_build_object( - 'maxCalculatedFieldsPerEntity', COALESCE(profile_data->'configuration'->>'maxCalculatedFieldsPerEntity', '5')::bigint, - 'maxArgumentsPerCF', COALESCE(profile_data->'configuration'->>'maxArgumentsPerCF', '10')::bigint, - 'maxDataPointsPerRollingArg', COALESCE(profile_data->'configuration'->>'maxDataPointsPerRollingArg', '1000')::bigint, - 'maxStateSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxStateSizeInKBytes', '32')::bigint, - 'maxSingleValueArgumentSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxSingleValueArgumentSizeInKBytes', '2')::bigint - ) - ) - WHERE profile_data->'configuration' IS NOT NULL; - -END IF; - END; -$$; +UPDATE tenant_profile +SET profile_data = profile_data + || jsonb_build_object( + 'configuration', profile_data->'configuration' || jsonb_build_object( + 'maxCalculatedFieldsPerEntity', COALESCE(profile_data->'configuration'->>'maxCalculatedFieldsPerEntity', '5')::bigint, + 'maxArgumentsPerCF', COALESCE(profile_data->'configuration'->>'maxArgumentsPerCF', '10')::bigint, + 'maxDataPointsPerRollingArg', COALESCE(profile_data->'configuration'->>'maxDataPointsPerRollingArg', '1000')::bigint, + 'maxStateSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxStateSizeInKBytes', '32')::bigint, + 'maxSingleValueArgumentSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxSingleValueArgumentSizeInKBytes', '2')::bigint + ) + ) +WHERE profile_data->'configuration'->>'maxCalculatedFieldsPerEntity' IS NULL; -- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS END From 3365d7f34e44515390a0e823181788ace3241ac7 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 5 Mar 2025 10:03:46 +0200 Subject: [PATCH 20/22] added option to round result for simple type --- .../ctx/state/SimpleCalculatedFieldState.java | 15 +++++++++++- .../state/SimpleCalculatedFieldStateTest.java | 24 ++++++++++++++++++- .../common/data/cf/configuration/Output.java | 1 + .../thingsboard/script/api/tbel/TbUtils.java | 7 +++++- .../script/api/tbel/TbUtilsTest.java | 7 ++++++ 5 files changed, 51 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 25008a60bd..480b334ac3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.kv.BasicKvEntry; @@ -64,7 +65,19 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { double expressionResult = expr.evaluate(); Output output = ctx.getOutput(); - return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), JacksonUtil.valueToTree(Map.of(output.getName(), expressionResult)))); + Object result; + Integer decimals = output.getDecimalsByDefault(); + if (decimals != null) { + if (decimals.equals(0)) { + result = TbUtils.toInt(expressionResult); + } else { + result = TbUtils.toFixed(expressionResult, decimals); + } + } else { + result = expressionResult; + } + + return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), JacksonUtil.valueToTree(Map.of(output.getName(), result)))); } } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index c06e835937..e179e8e706 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.dao.usagerecord.ApiLimitService; @@ -138,7 +139,7 @@ public class SimpleCalculatedFieldStateTest { Output output = getCalculatedFieldConfig().getOutput(); assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49.0))); + assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49))); } @Test @@ -154,6 +155,26 @@ public class SimpleCalculatedFieldStateTest { .hasMessage("Argument 'key2' is not a number."); } + @Test + void testPerformCalculationWhenDecimalsByDefault() throws ExecutionException, InterruptedException { + state.arguments = new HashMap<>(Map.of( + "key1", new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new DoubleDataEntry("key1", 11.3456), 145L), + "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 6, new DoubleDataEntry("key2", 15.1), 165L), + "key3", new SingleValueArgumentEntry(System.currentTimeMillis() - 3, new DoubleDataEntry("key3", 23.1), 184L) + )); + + Output output = getCalculatedFieldConfig().getOutput(); + output.setDecimalsByDefault(3); + ctx.setOutput(output); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49.546))); + } + @Test void testIsReadyWhenNotAllArgPresent() { assertThat(state.isReady()).isFalse(); @@ -219,6 +240,7 @@ public class SimpleCalculatedFieldStateTest { output.setName("output"); output.setType(OutputType.ATTRIBUTES); output.setScope(AttributeScope.SERVER_SCOPE); + output.setDecimalsByDefault(0); config.setOutput(output); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java index 49e393b19f..f2b4948837 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -26,5 +26,6 @@ public class Output { private String name; private OutputType type; private AttributeScope scope; + private Integer decimalsByDefault; } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index a43cc30f23..20a753da14 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -44,7 +44,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeSet; import java.util.regex.Matcher; import static java.lang.Character.MAX_RADIX; @@ -256,6 +255,8 @@ public class TbUtils { double.class, int.class))); parserConfig.addImport("toFixed", new MethodStub(TbUtils.class.getMethod("toFixed", float.class, int.class))); + parserConfig.addImport("toInt", new MethodStub(TbUtils.class.getMethod("toInt", + double.class))); parserConfig.addImport("hexToBytes", new MethodStub(TbUtils.class.getMethod("hexToBytes", ExecutionContext.class, String.class))); parserConfig.addImport("hexToBytesArray", new MethodStub(TbUtils.class.getMethod("hexToBytesArray", @@ -1156,6 +1157,10 @@ public class TbUtils { return BigDecimal.valueOf(value).setScale(precision, RoundingMode.HALF_UP).floatValue(); } + public static int toInt(double value) { + return BigDecimal.valueOf(value).setScale(0, RoundingMode.HALF_UP).intValue(); + } + public static ExecutionHashMap toFlatMap(ExecutionContext ctx, Map json) { return toFlatMap(ctx, json, new ArrayList<>(), true); } diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index 71c7a8b78a..69987d3132 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -1137,6 +1137,13 @@ public class TbUtilsTest { Assertions.assertEquals(expected, actual); } + @Test + void toInt() { + Assertions.assertEquals(1729, TbUtils.toInt(doubleVal)); + Assertions.assertEquals(13, TbUtils.toInt(12.8)); + Assertions.assertEquals(28, TbUtils.toInt(28.0)); + } + private static List toList(byte[] data) { List result = new ArrayList<>(data.length); for (Byte b : data) { From 622580c97cbdec03d47d0e6e0431d20d3e152cdd Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 5 Mar 2025 12:48:59 +0200 Subject: [PATCH 21/22] Update GitHub workflow Ubuntu version to 22.04 --- .github/workflows/check-configuration-files.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-configuration-files.yml b/.github/workflows/check-configuration-files.yml index f280e326c2..21357a8a3c 100644 --- a/.github/workflows/check-configuration-files.yml +++ b/.github/workflows/check-configuration-files.yml @@ -32,7 +32,7 @@ on: jobs: build: name: Check thingsboard.yml file - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Checkout code uses: actions/checkout@v2 From c990169223b52f4c90b66d81fcc65ab93c098e20 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 5 Mar 2025 13:07:38 +0200 Subject: [PATCH 22/22] Update GitHub workflow Python version to 3.13.2 --- .github/workflows/check-configuration-files.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-configuration-files.yml b/.github/workflows/check-configuration-files.yml index 21357a8a3c..561b7d0019 100644 --- a/.github/workflows/check-configuration-files.yml +++ b/.github/workflows/check-configuration-files.yml @@ -36,10 +36,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Set up Python 3.10 + - name: Set up Python 3.13 uses: actions/setup-python@v3 with: - python-version: "3.10.2" + python-version: "3.13.2" architecture: "x64" env: AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache