From a18e76839bdff4843b9130e96bc397dba1f88638 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 15 Apr 2025 15:40:50 +0300 Subject: [PATCH 01/56] added calculated fields to rest client --- .../thingsboard/rest/client/RestClient.java | 65 ++++++++++++++++++- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 4f7538d205..606107b92f 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -55,9 +55,9 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; import org.thingsboard.server.common.data.EventInfo; -import org.thingsboard.server.common.data.ResourceExportData; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.OtaPackageInfo; +import org.thingsboard.server.common.data.ResourceExportData; import org.thingsboard.server.common.data.ResourceSubType; import org.thingsboard.server.common.data.SaveDeviceWithCredentialsRequest; import org.thingsboard.server.common.data.StringUtils; @@ -86,6 +86,7 @@ import org.thingsboard.server.common.data.asset.AssetProfileInfo; import org.thingsboard.server.common.data.asset.AssetSearchQuery; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.audit.AuditLog; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.device.DeviceSearchQuery; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.domain.DomainInfo; @@ -99,6 +100,7 @@ import org.thingsboard.server.common.data.id.AlarmCommentId; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceId; @@ -3765,7 +3767,7 @@ public class RestClient implements Closeable { } public PageData getImages(PageLink pageLink, boolean includeSystemImages) { - return this.getImages(pageLink, null, includeSystemImages); + return this.getImages(pageLink, null, includeSystemImages); } public PageData getImages(PageLink pageLink, ResourceSubType imageSubType, boolean includeSystemImages) { @@ -4056,6 +4058,65 @@ public class RestClient implements Closeable { timeout).getBody(); } + public CalculatedField saveCalculatedField(CalculatedField calculatedField) { + return restTemplate.postForEntity(baseURL + "/api/calculatedField", calculatedField, CalculatedField.class).getBody(); + } + + public Optional getCalculatedFieldById(CalculatedFieldId calculatedFieldId) { + try { + ResponseEntity calculatedField = restTemplate.getForEntity(baseURL + "/api/calculatedField/{calculatedFieldId}", CalculatedField.class, calculatedFieldId.getId()); + return Optional.ofNullable(calculatedField.getBody()); + } catch (HttpClientErrorException exception) { + if (exception.getStatusCode() == HttpStatus.NOT_FOUND) { + return Optional.empty(); + } else { + throw exception; + } + } + } + + public PageData getCalculatedFieldsByEntityId(EntityId entityId, PageLink pageLink) { + Map params = new HashMap<>(); + params.put("entityType", entityId.getEntityType().name()); + params.put("entityId", entityId.getId().toString()); + addPageLinkToParam(params, pageLink); + return restTemplate.exchange( + baseURL + "/api/{entityType}/{entityId}/calculatedFields?" + getUrlParams(pageLink), + HttpMethod.GET, HttpEntity.EMPTY, + new ParameterizedTypeReference>() { + }, params).getBody(); + } + + public void deleteCalculatedField(CalculatedFieldId calculatedFieldId) { + restTemplate.delete(baseURL + "/api/calculatedField/{calculatedFieldId}", calculatedFieldId.getId()); + } + + public Optional getLatestCalculatedFieldDebugEvent(CalculatedFieldId calculatedFieldId) { + try { + ResponseEntity jsonNode = restTemplate.getForEntity(baseURL + "/api/calculatedField/{calculatedFieldId}/debug", JsonNode.class, calculatedFieldId.getId()); + return Optional.ofNullable(jsonNode.getBody()); + } catch (HttpClientErrorException exception) { + if (exception.getStatusCode() == HttpStatus.NOT_FOUND) { + return Optional.empty(); + } else { + throw exception; + } + } + } + + public Optional testCalculatedFieldScript(JsonNode inputParams) { + try { + ResponseEntity jsonNode = restTemplate.postForEntity(baseURL + "/api/calculatedField/testScript", inputParams, JsonNode.class); + return Optional.ofNullable(jsonNode.getBody()); + } catch (HttpClientErrorException exception) { + if (exception.getStatusCode() == HttpStatus.NOT_FOUND) { + return Optional.empty(); + } else { + throw exception; + } + } + } + private String getTimeUrlParams(TimePageLink pageLink) { String urlParams = getUrlParams(pageLink); if (pageLink.getStartTime() != null) { From a2382c206f10d84c17d0336bb9cc086ecf19d549 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 23 May 2025 16:52:03 +0300 Subject: [PATCH 02/56] fixes for performance --- .../server/actors/ActorSystemContext.java | 4 -- .../cf/DefaultCalculatedFieldCache.java | 29 +++++++++----- .../DefaultCalculatedFieldQueueService.java | 40 +++++++++++++------ .../cf/ctx/state/CalculatedFieldCtx.java | 33 +++++++++------ .../KafkaCalculatedFieldStateService.java | 19 +-------- .../queue/DefaultTbClusterService.java | 2 - .../sqlts/dictionary/JpaKeyDictionaryDao.java | 2 - .../sql/SqlLatestInsertTsRepository.java | 2 - .../thingsboard/server/dao/util/KvUtils.java | 15 +++---- 9 files changed, 78 insertions(+), 68 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 61d9586095..bd45aa404f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -318,7 +318,6 @@ public class ActorSystemContext { @Getter private TbEntityViewService tbEntityViewService; - @Lazy @Autowired @Getter private TelemetrySubscriptionService tsSubService; @@ -537,17 +536,14 @@ public class ActorSystemContext { @Getter private EntityService entityService; - @Lazy @Autowired(required = false) @Getter private CalculatedFieldProcessingService calculatedFieldProcessingService; - @Lazy @Autowired(required = false) @Getter private CalculatedFieldStateService calculatedFieldStateService; - @Lazy @Autowired(required = false) @Getter private CalculatedFieldQueueService calculatedFieldQueueService; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index ed35d96cb7..383c019858 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -18,8 +18,11 @@ package org.thingsboard.server.service.cf; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; +import org.springframework.util.ConcurrentReferenceHashMap; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.cf.CalculatedField; @@ -49,14 +52,14 @@ import java.util.concurrent.locks.ReentrantLock; @RequiredArgsConstructor public class DefaultCalculatedFieldCache implements CalculatedFieldCache { - private static final Integer UNKNOWN_PARTITION = -1; - - private final Lock calculatedFieldFetchLock = new ReentrantLock(); + private final ConcurrentReferenceHashMap calculatedFieldFetchLocks = new ConcurrentReferenceHashMap<>(); private final CalculatedFieldService calculatedFieldService; private final TbelInvokeService tbelInvokeService; - private final ActorSystemContext actorSystemContext; private final ApiLimitService apiLimitService; + @Autowired + @Lazy + private ActorSystemContext actorSystemContext; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); @@ -98,19 +101,20 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public List getCalculatedFieldsByEntityId(EntityId entityId) { - return entityIdCalculatedFields.getOrDefault(entityId, new CopyOnWriteArrayList<>()); + return entityIdCalculatedFields.getOrDefault(entityId, Collections.emptyList()); } @Override public List getCalculatedFieldLinksByEntityId(EntityId entityId) { - return entityIdCalculatedFieldLinks.getOrDefault(entityId, new CopyOnWriteArrayList<>()); + return entityIdCalculatedFieldLinks.getOrDefault(entityId, Collections.emptyList()); } @Override public CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId) { CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); if (ctx == null) { - calculatedFieldFetchLock.lock(); + Lock lock = getFetchLock(calculatedFieldId); + lock.lock(); try { ctx = calculatedFieldsCtx.get(calculatedFieldId); if (ctx == null) { @@ -122,13 +126,17 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } } } finally { - calculatedFieldFetchLock.unlock(); + lock.unlock(); } } log.trace("[{}] Found calculated field ctx in cache: {}", calculatedFieldId, ctx); return ctx; } + private Lock getFetchLock(CalculatedFieldId id) { + return calculatedFieldFetchLocks.computeIfAbsent(id, __ -> new ReentrantLock()); + } + @Override public List getCalculatedFieldCtxsByEntityId(EntityId entityId) { if (entityId == null) { @@ -141,7 +149,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - calculatedFieldFetchLock.lock(); + Lock lock = getFetchLock(calculatedFieldId); + lock.lock(); try { CalculatedField calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); if (calculatedField == null) { @@ -163,7 +172,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { .add(configuration.buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)); }); } finally { - calculatedFieldFetchLock.unlock(); + lock.unlock(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 8289e4db42..8b724a65ac 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -133,7 +133,8 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); if (send) { - clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); + ToCalculatedFieldMsg calculatedFieldMsg = msg.get(); + clusterService.pushMsgToCalculatedFields(tenantId, entityId, calculatedFieldMsg, wrap(callback)); } else { if (callback != null) { callback.onSuccess(null); @@ -142,20 +143,35 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter) { - boolean send = false; - if (supportedReferencedEntities.contains(entityId.getEntityType())) { - send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId).stream().anyMatch(filter); - if (!send) { - send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(getProfileId(tenantId, entityId)).stream().anyMatch(filter); + if (!supportedReferencedEntities.contains(entityId.getEntityType())) { + return false; + } + List entityCfs = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId); + for (CalculatedFieldCtx ctx : entityCfs) { + if (filter.test(ctx)) { + return true; } - if (!send) { - send = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId).stream() - .map(CalculatedFieldLink::getCalculatedFieldId) - .map(calculatedFieldCache::getCalculatedFieldCtx) - .anyMatch(linkedEntityFilter); + } + + EntityId profileId = getProfileId(tenantId, entityId); + if (profileId != null) { + List profileCfs = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(profileId); + for (CalculatedFieldCtx ctx : profileCfs) { + if (filter.test(ctx)) { + return true; + } } } - return send; + + List links = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId); + for (CalculatedFieldLink link : links) { + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(link.getCalculatedFieldId()); + if (ctx != null && linkedEntityFilter.test(ctx)) { + return true; + } + } + + return false; } private EntityId getProfileId(TenantId tenantId, EntityId entityId) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index a3fdae319d..6bcedd61e1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -181,6 +181,10 @@ public class CalculatedFieldCtx { } private boolean matchesAttributes(Map argMap, List values, AttributeScope scope) { + if (argMap.isEmpty() || values.isEmpty()) { + return false; + } + for (AttributeKvEntry attrKv : values) { ReferencedEntityKey attrKey = new ReferencedEntityKey(attrKv.getKey(), ArgumentType.ATTRIBUTE, scope); if (argMap.containsKey(attrKey)) { @@ -191,13 +195,14 @@ public class CalculatedFieldCtx { } private boolean matchesTimeSeries(Map argMap, List values) { + if (argMap.isEmpty() || values.isEmpty()) { + return false; + } + for (TsKvEntry tsKv : values) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_LATEST, null); - if (argMap.containsKey(latestKey)) { - return true; - } - ReferencedEntityKey rollingKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_ROLLING, null); - if (argMap.containsKey(rollingKey)) { + String key = tsKv.getKey(); + if (argMap.containsKey(new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null)) || + argMap.containsKey(new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null))) { return true; } } @@ -213,6 +218,10 @@ public class CalculatedFieldCtx { } private boolean matchesAttributesKeys(Map argMap, List keys, AttributeScope scope) { + if (argMap.isEmpty() || keys.isEmpty()) { + return false; + } + for (String key : keys) { ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); if (argMap.containsKey(attrKey)) { @@ -223,13 +232,13 @@ public class CalculatedFieldCtx { } private boolean matchesTimeSeriesKeys(Map argMap, List keys) { + if (argMap.isEmpty() || keys.isEmpty()) { + return false; + } + for (String key : keys) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); - if (argMap.containsKey(latestKey)) { - return true; - } - ReferencedEntityKey rollingKey = new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null); - if (argMap.containsKey(rollingKey)) { + if (argMap.containsKey(new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null)) || + argMap.containsKey(new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null))) { return true; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index 533b487d38..36ae4fd334 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -30,9 +30,7 @@ import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; -import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgHeaders; -import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; import org.thingsboard.server.queue.common.state.KafkaQueueStateService; @@ -111,21 +109,8 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta if (stateMsgProto == null) { putStateId(msg.getHeaders(), stateId); } - stateProducer.send(tpi, stateId.toKey(), msg, new TbQueueCallback() { - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - if (callback != null) { - callback.onSuccess(); - } - } - - @Override - public void onFailure(Throwable t) { - if (callback != null) { - callback.onFailure(t); - } - } - }); + stateProducer.send(tpi, stateId.toKey(), msg, null); + callback.onSuccess(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 561dc7122a..b5f7669fe1 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -130,11 +130,9 @@ public class DefaultTbClusterService implements TbClusterService { private final AtomicInteger toEdgeNfs = new AtomicInteger(0); @Autowired - @Lazy private PartitionService partitionService; @Autowired - @Lazy private TbQueueProducerProvider producerProvider; @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java index 53a824f026..c14c069f23 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java @@ -29,7 +29,6 @@ import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; -import org.thingsboard.server.dao.util.SqlDao; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; @@ -38,7 +37,6 @@ import java.util.concurrent.locks.ReentrantLock; @Component @Slf4j -@SqlDao @RequiredArgsConstructor public class JpaKeyDictionaryDao extends JpaAbstractDaoListeningExecutorService implements KeyDictionaryDao { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java index b40fa520f2..63afd2bd26 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java @@ -22,7 +22,6 @@ import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.AbstractVersionedInsertRepository; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; -import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; import java.sql.PreparedStatement; @@ -33,7 +32,6 @@ import java.util.List; @SqlTsLatestAnyDao @Repository @Transactional -@SqlDao public class SqlLatestInsertTsRepository extends AbstractVersionedInsertRepository implements InsertLatestTsRepository { @Value("${sql.ts_latest.update_by_latest_ts:true}") diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/KvUtils.java b/dao/src/main/java/org/thingsboard/server/dao/util/KvUtils.java index eeb88959fa..8b95ddcb57 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/KvUtils.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/KvUtils.java @@ -33,9 +33,8 @@ public class KvUtils { static { validatedKeys = Caffeine.newBuilder() - .weakKeys() .expireAfterAccess(24, TimeUnit.HOURS) - .maximumSize(100000).build(); + .maximumSize(50000).build(); } public static void validate(List tsKvEntries, boolean valueNoXssValidation) { @@ -57,11 +56,13 @@ public class KvUtils { throw new DataValidationException("Validation error: key length must be equal or less than 255"); } - if (validatedKeys.getIfPresent(key) == null) { - if (!NoXssValidator.isValid(key)) { - throw new DataValidationException("Validation error: key is malformed"); - } - validatedKeys.put(key, Boolean.TRUE); + Boolean isValid = validatedKeys.asMap().get(key); + if (isValid == null) { + isValid = NoXssValidator.isValid(key); + validatedKeys.put(key, isValid); + } + if (!isValid) { + throw new DataValidationException("Validation error: key is malformed"); } if (valueNoXssValidation) { From b20c33cd1d34f6478ab8e1f818b185eb4e94612f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 4 Jun 2025 15:14:13 +0300 Subject: [PATCH 03/56] moved method --- .../service/cf/DefaultCalculatedFieldCache.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 383c019858..3c45f3e921 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.cf; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; @@ -57,9 +56,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final CalculatedFieldService calculatedFieldService; private final TbelInvokeService tbelInvokeService; private final ApiLimitService apiLimitService; - @Autowired @Lazy - private ActorSystemContext actorSystemContext; + private final ActorSystemContext actorSystemContext; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); @@ -133,10 +131,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return ctx; } - private Lock getFetchLock(CalculatedFieldId id) { - return calculatedFieldFetchLocks.computeIfAbsent(id, __ -> new ReentrantLock()); - } - @Override public List getCalculatedFieldCtxsByEntityId(EntityId entityId) { if (entityId == null) { @@ -196,4 +190,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); } + private Lock getFetchLock(CalculatedFieldId id) { + return calculatedFieldFetchLocks.computeIfAbsent(id, __ -> new ReentrantLock()); + } + } From edd9a6a552d590ec129416a225edf8193c11c2cc Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 6 Jun 2025 16:38:28 +0300 Subject: [PATCH 04/56] removed unnecessary lazy --- .../server/actors/ActorSystemContext.java | 4 +- .../DefaultCalculatedFieldQueueService.java | 33 +++++++++------- .../cf/ctx/state/CalculatedFieldCtx.java | 39 ++++++++++++++----- .../AttributeKvInsertRepository.java | 4 -- .../sql/SqlLatestInsertTsRepository.java | 2 - 5 files changed, 51 insertions(+), 31 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index a3cd5de7bd..0125c7c07d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -81,6 +81,7 @@ import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.dao.job.JobService; import org.thingsboard.server.dao.mobile.MobileAppBundleService; import org.thingsboard.server.dao.mobile.MobileAppService; import org.thingsboard.server.dao.nosql.CassandraBufferedRateReadExecutor; @@ -97,7 +98,6 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.rule.RuleNodeStateService; -import org.thingsboard.server.dao.job.JobService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.tenant.TenantService; @@ -452,12 +452,10 @@ public class ActorSystemContext { @Getter private ApiLimitService apiLimitService; - @Lazy @Autowired(required = false) @Getter private RateLimitService rateLimitService; - @Lazy @Autowired(required = false) @Getter private DebugModeRateLimitsConfig debugModeRateLimitsConfig; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 8b724a65ac..bf206bc5dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -40,7 +40,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; @@ -86,7 +85,10 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); - checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), + var entries = request.getEntries(); + checkEntityAndPushToQueue(tenantId, entityId, + cf -> cf.matches(entries), + cf -> cf.linkMatches(entityId, entries), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -99,7 +101,11 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS public void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); - checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), + var entries = request.getEntries(); + var scope = request.getScope(); + checkEntityAndPushToQueue(tenantId, entityId, + cf -> cf.matches(entries, scope), + cf -> cf.linkMatches(entityId, entries, scope), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -112,7 +118,10 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS public void pushRequestToQueue(AttributesDeleteRequest request, List result, FutureCallback callback) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); - checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matchesKeys(result, request.getScope()), cf -> cf.linkMatchesAttrKeys(entityId, result, request.getScope()), + var scope = request.getScope(); + checkEntityAndPushToQueue(tenantId, entityId, + cf -> cf.matchesKeys(result, scope), + cf -> cf.linkMatchesAttrKeys(entityId, result, scope), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -120,8 +129,9 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS public void pushRequestToQueue(TimeseriesDeleteRequest request, List result, FutureCallback callback) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); - - checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matchesKeys(result), cf -> cf.linkMatchesTsKeys(entityId, result), + checkEntityAndPushToQueue(tenantId, entityId, + cf -> cf.matchesKeys(result), + cf -> cf.linkMatchesTsKeys(entityId, result), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -183,23 +193,20 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { - ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); - CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); List entries = request.getEntries(); List versions = result != null ? result.getVersions() : Collections.emptyList(); for (int i = 0; i < entries.size(); i++) { - TsKvProto.Builder tsProtoBuilder = toTsKvProto(entries.get(i)).toBuilder(); + TsKvEntry tsKvEntry = entries.get(i); if (result != null) { - tsProtoBuilder.setVersion(versions.get(i)); + tsKvEntry.setVersion(versions.get(i)); } - telemetryMsg.addTsData(tsProtoBuilder.build()); + telemetryMsg.addTsData(toTsKvProto(tsKvEntry)); } - msg.setTelemetryMsg(telemetryMsg.build()); - return msg.build(); + return ToCalculatedFieldMsg.newBuilder().setTelemetryMsg(telemetryMsg).build(); } private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List versions) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 6bcedd61e1..b6c04426ba 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -185,12 +185,14 @@ public class CalculatedFieldCtx { return false; } + var lookupKey = new ReferencedEntityKey(null, ArgumentType.ATTRIBUTE, scope); for (AttributeKvEntry attrKv : values) { - ReferencedEntityKey attrKey = new ReferencedEntityKey(attrKv.getKey(), ArgumentType.ATTRIBUTE, scope); - if (argMap.containsKey(attrKey)) { + lookupKey.setKey(attrKv.getKey()); + if (argMap.containsKey(lookupKey)) { return true; } } + return false; } @@ -199,13 +201,21 @@ public class CalculatedFieldCtx { return false; } + var lookupKey = new ReferencedEntityKey(null, null, null); for (TsKvEntry tsKv : values) { - String key = tsKv.getKey(); - if (argMap.containsKey(new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null)) || - argMap.containsKey(new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null))) { + lookupKey.setKey(tsKv.getKey()); + + lookupKey.setType(ArgumentType.TS_LATEST); + if (argMap.containsKey(lookupKey)) { + return true; + } + + lookupKey.setType(ArgumentType.TS_ROLLING); + if (argMap.containsKey(lookupKey)) { return true; } } + return false; } @@ -222,12 +232,14 @@ public class CalculatedFieldCtx { return false; } + var lookupKey = new ReferencedEntityKey(null, ArgumentType.ATTRIBUTE, scope); for (String key : keys) { - ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); - if (argMap.containsKey(attrKey)) { + lookupKey.setKey(key); + if (argMap.containsKey(lookupKey)) { return true; } } + return false; } @@ -236,12 +248,21 @@ public class CalculatedFieldCtx { return false; } + var lookupKey = new ReferencedEntityKey(null, null, null); for (String key : keys) { - if (argMap.containsKey(new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null)) || - argMap.containsKey(new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null))) { + lookupKey.setKey(key); + + lookupKey.setType(ArgumentType.TS_LATEST); + if (argMap.containsKey(lookupKey)) { + return true; + } + + lookupKey.setType(ArgumentType.TS_ROLLING); + if (argMap.containsKey(lookupKey)) { return true; } } + return false; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java index 48a1eb6d5a..801c436d2f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java @@ -16,10 +16,8 @@ package org.thingsboard.server.dao.sql.attributes; import org.springframework.stereotype.Repository; -import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.AbstractVersionedInsertRepository; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -27,8 +25,6 @@ import java.sql.Types; import java.util.List; @Repository -@Transactional -@SqlDao public class AttributeKvInsertRepository extends AbstractVersionedInsertRepository { private static final String BATCH_UPDATE = "UPDATE attribute_kv SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ?, version = nextval('attribute_kv_version_seq') " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java index 63afd2bd26..c9d16f00ec 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java @@ -18,7 +18,6 @@ package org.thingsboard.server.dao.sqlts.insert.latest.sql; import jakarta.annotation.PostConstruct; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Repository; -import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.AbstractVersionedInsertRepository; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; @@ -31,7 +30,6 @@ import java.util.List; @SqlTsLatestAnyDao @Repository -@Transactional public class SqlLatestInsertTsRepository extends AbstractVersionedInsertRepository implements InsertLatestTsRepository { @Value("${sql.ts_latest.update_by_latest_ts:true}") From 8dc9a68c625b34ad7523aea6bdfe5817fffc4b12 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Fri, 6 Jun 2025 18:47:45 +0300 Subject: [PATCH 05/56] Update cached activity status only after a successful database save --- .../state/DefaultDeviceStateService.java | 97 ++-- .../DefaultTelemetrySubscriptionService.java | 4 +- .../telemetry/InternalTelemetryService.java | 4 +- .../src/main/resources/thingsboard.yml | 2 + .../state/DefaultDeviceStateServiceTest.java | 527 ++++++++---------- 5 files changed, 283 insertions(+), 351 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index cc476d377d..f46323f702 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -27,11 +27,10 @@ import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; -import lombok.Getter; import lombok.RequiredArgsConstructor; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; +import org.checkerframework.checker.nullness.qual.NonNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; @@ -170,35 +169,22 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService> stats = new HashMap<>(); for (DeviceStateData stateData : deviceStates.values()) { @@ -587,13 +572,12 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService { + stateData.getState().setActive(active); + pushRuleEngineMessage(stateData, active ? TbMsgType.ACTIVITY_EVENT : TbMsgType.INACTIVITY_EVENT); + TbMsgMetaData metaData = stateData.getMetaData(); + notificationRuleProcessor.process(DeviceActivityTrigger.builder() + .tenantId(tenantId) + .customerId(stateData.getCustomerId()) + .deviceId(deviceId) + .active(active) + .deviceName(metaData.getValue("deviceName")) + .deviceType(metaData.getValue("deviceType")) + .deviceLabel(metaData.getValue("deviceLabel")) + .build()); + }, deviceStateCallbackExecutor); } boolean cleanDeviceStateIfBelongsToExternalPartition(TenantId tenantId, final DeviceId deviceId) { @@ -634,8 +625,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService save(TenantId tenantId, DeviceId deviceId, String key, long value) { + return save(tenantId, deviceId, new LongDataEntry(key, value), getCurrentTimeMillis()); } - private void save(TenantId tenantId, DeviceId deviceId, String key, boolean value) { - save(tenantId, deviceId, new BooleanDataEntry(key, value), getCurrentTimeMillis()); + private ListenableFuture save(TenantId tenantId, DeviceId deviceId, String key, boolean value) { + return save(tenantId, deviceId, new BooleanDataEntry(key, value), getCurrentTimeMillis()); } - private void save(TenantId tenantId, DeviceId deviceId, KvEntry kvEntry, long ts) { + private ListenableFuture save(TenantId tenantId, DeviceId deviceId, KvEntry kvEntry, long ts) { + ListenableFuture future; if (persistToTelemetry) { - tsSubService.saveTimeseriesInternal(TimeseriesSaveRequest.builder() + future = tsSubService.saveTimeseriesInternal(TimeseriesSaveRequest.builder() .tenantId(tenantId) .entityId(deviceId) .entry(new BasicTsKvEntry(ts, kvEntry)) @@ -895,7 +889,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService(deviceId, kvEntry)) .build()); } else { - tsSubService.saveAttributes(AttributesSaveRequest.builder() + future = tsSubService.saveAttributesInternal(AttributesSaveRequest.builder() .tenantId(tenantId) .entityId(deviceId) .scope(AttributeScope.SERVER_SCOPE) @@ -903,20 +897,14 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService(deviceId, kvEntry)) .build()); } + return Futures.transform(future, __ -> null, MoreExecutors.directExecutor()); } long getCurrentTimeMillis() { return System.currentTimeMillis(); } - private static class TelemetrySaveCallback implements FutureCallback { - private final DeviceId deviceId; - private final KvEntry kvEntry; - - TelemetrySaveCallback(DeviceId deviceId, KvEntry kvEntry) { - this.deviceId = deviceId; - this.kvEntry = kvEntry; - } + private record TelemetrySaveCallback(DeviceId deviceId, KvEntry kvEntry) implements FutureCallback { @Override public void onSuccess(@Nullable T result) { @@ -924,9 +912,10 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService> saveAttributesInternal(AttributesSaveRequest request) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); AttributesSaveRequest.Strategy strategy = request.getStrategy(); @@ -228,6 +227,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (strategy.sendWsUpdate()) { addWsCallback(resultFuture, success -> onAttributesUpdate(tenantId, entityId, request.getScope().name(), request.getEntries())); } + return resultFuture; } private static boolean shouldSendSharedAttributesUpdatedNotification(AttributesSaveRequest request) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java index 8a76aa1d14..79f0beab41 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java @@ -23,6 +23,8 @@ import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; +import java.util.List; + /** * Created by ashvayka on 27.03.18. */ @@ -30,7 +32,7 @@ public interface InternalTelemetryService extends RuleEngineTelemetryService { ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request); - void saveAttributesInternal(AttributesSaveRequest request); + ListenableFuture> saveAttributesInternal(AttributesSaveRequest request); void deleteTimeseriesInternal(TimeseriesDeleteRequest request); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 6d27b9bd7e..09820a81cd 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -898,6 +898,8 @@ state: # Used only when state.persistToTelemetry is set to 'true' and Cassandra is used for timeseries data. # 0 means time-to-live mechanism is disabled. telemetryTtl: "${STATE_TELEMETRY_TTL:0}" + # Number of device records to fetch per batch when initializing device activity states + initFetchPackSize: "${TB_DEVICE_STATE_INIT_FETCH_PACK_SIZE:50000}" # Configuration properties for rule nodes related to device activity state rule: node: diff --git a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java index 26e913eacc..cbf7363441 100644 --- a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java @@ -16,6 +16,9 @@ package org.thingsboard.server.service.state; import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -31,13 +34,12 @@ import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceIdInfo; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.notification.rule.trigger.DeviceActivityTrigger; -import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; @@ -50,20 +52,22 @@ import org.thingsboard.server.dao.sql.query.EntityQueryRepository; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.QueueKey; -import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.usagestats.DefaultTbApiUsageReportClient; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; +import java.time.Duration; import java.util.Collections; +import java.util.HashSet; import java.util.List; -import java.util.Map; +import java.util.Set; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -77,8 +81,8 @@ import static org.mockito.BDDMockito.then; import static org.mockito.BDDMockito.willReturn; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.never; -import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -90,7 +94,10 @@ import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAS import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_DISCONNECT_TIME; @ExtendWith(MockitoExtension.class) -public class DefaultDeviceStateServiceTest { +class DefaultDeviceStateServiceTest { + + ListeningExecutorService deviceStateExecutor; + ListeningExecutorService deviceStateCallbackExecutor; @Mock DeviceService deviceService; @@ -113,25 +120,48 @@ public class DefaultDeviceStateServiceTest { @Mock DefaultTbApiUsageReportClient defaultTbApiUsageReportClient; - TenantId tenantId = new TenantId(UUID.fromString("00797a3b-7aeb-4b5b-b57a-c2a810d0f112")); - DeviceId deviceId = DeviceId.fromString("00797a3b-7aeb-4b5b-b57a-c2a810d0f112"); - TopicPartitionInfo tpi; + long defaultInactivityTimeoutMs = Duration.ofMinutes(10L).toMillis(); + + TenantId tenantId = TenantId.fromUUID(UUID.fromString("00797a3b-7aeb-4b5b-b57a-c2a810d0f112")); + DeviceId deviceId = DeviceId.fromString("c209f718-42e5-11f0-9fe2-0242ac120002"); + TopicPartitionInfo tpi = TopicPartitionInfo.builder() + .topic("tb_core") + .partition(0) + .myPartition(true) + .build(); DefaultDeviceStateService service; @BeforeEach - public void setUp() { + void setUp() { service = spy(new DefaultDeviceStateService(deviceService, attributesService, tsService, clusterService, partitionService, entityQueryRepository, null, defaultTbApiUsageReportClient, notificationRuleProcessor)); ReflectionTestUtils.setField(service, "tsSubService", telemetrySubscriptionService); + ReflectionTestUtils.setField(service, "defaultInactivityTimeoutMs", defaultInactivityTimeoutMs); ReflectionTestUtils.setField(service, "defaultStateCheckIntervalInSec", 60); ReflectionTestUtils.setField(service, "defaultActivityStatsIntervalInSec", 60); - ReflectionTestUtils.setField(service, "initFetchPackSize", 10); + ReflectionTestUtils.setField(service, "initFetchPackSize", 50000); + + deviceStateExecutor = MoreExecutors.newDirectExecutorService(); + ReflectionTestUtils.setField(service, "deviceStateExecutor", deviceStateExecutor); + + deviceStateCallbackExecutor = MoreExecutors.newDirectExecutorService(); + ReflectionTestUtils.setField(service, "deviceStateCallbackExecutor", deviceStateCallbackExecutor); + + lenient().when(partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId)).thenReturn(tpi); + + ConcurrentMap> partitionedEntities = new ConcurrentHashMap<>(); + partitionedEntities.put(tpi, new HashSet<>()); + ReflectionTestUtils.setField(service, "partitionedEntities", partitionedEntities); + } - tpi = TopicPartitionInfo.builder().myPartition(true).build(); + @AfterEach + void cleanup() { + deviceStateExecutor.shutdownNow(); + deviceStateCallbackExecutor.shutdownNow(); } @Test - public void givenDeviceBelongsToExternalPartition_whenOnDeviceConnect_thenCleansStateAndDoesNotReportConnect() { + void givenDeviceBelongsToExternalPartition_whenOnDeviceConnect_thenCleansStateAndDoesNotReportConnect() { // GIVEN doReturn(true).when(service).cleanDeviceStateIfBelongsToExternalPartition(tenantId, deviceId); @@ -149,7 +179,7 @@ public class DefaultDeviceStateServiceTest { @ParameterizedTest @ValueSource(longs = {Long.MIN_VALUE, -100, -1}) - public void givenNegativeLastConnectTime_whenOnDeviceConnect_thenSkipsThisEvent(long negativeLastConnectTime) { + void givenNegativeLastConnectTime_whenOnDeviceConnect_thenSkipsThisEvent(long negativeLastConnectTime) { // GIVEN doReturn(false).when(service).cleanDeviceStateIfBelongsToExternalPartition(tenantId, deviceId); @@ -166,7 +196,7 @@ public class DefaultDeviceStateServiceTest { @ParameterizedTest @MethodSource("provideOutdatedTimestamps") - public void givenOutdatedLastConnectTime_whenOnDeviceDisconnect_thenSkipsThisEvent(long outdatedLastConnectTime, long currentLastConnectTime) { + void givenOutdatedLastConnectTime_whenOnDeviceDisconnect_thenSkipsThisEvent(long outdatedLastConnectTime, long currentLastConnectTime) { // GIVEN doReturn(false).when(service).cleanDeviceStateIfBelongsToExternalPartition(tenantId, deviceId); @@ -188,7 +218,7 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceBelongsToMyPartition_whenOnDeviceConnect_thenReportsConnect() { + void givenDeviceBelongsToMyPartition_whenOnDeviceConnect_thenReportsConnect() { // GIVEN var deviceStateData = DeviceStateData.builder() .tenantId(tenantId) @@ -202,11 +232,13 @@ public class DefaultDeviceStateServiceTest { service.deviceStates.put(deviceId, deviceStateData); long lastConnectTime = System.currentTimeMillis(); + mockSuccessfulSaveAttributes(); + // WHEN service.onDeviceConnect(tenantId, deviceId, lastConnectTime); // THEN - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(LAST_CONNECT_TIME) && @@ -221,7 +253,7 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceBelongsToExternalPartition_whenOnDeviceDisconnect_thenCleansStateAndDoesNotReportDisconnect() { + void givenDeviceBelongsToExternalPartition_whenOnDeviceDisconnect_thenCleansStateAndDoesNotReportDisconnect() { // GIVEN doReturn(true).when(service).cleanDeviceStateIfBelongsToExternalPartition(tenantId, deviceId); @@ -238,7 +270,7 @@ public class DefaultDeviceStateServiceTest { @ParameterizedTest @ValueSource(longs = {Long.MIN_VALUE, -100, -1}) - public void givenNegativeLastDisconnectTime_whenOnDeviceDisconnect_thenSkipsThisEvent(long negativeLastDisconnectTime) { + void givenNegativeLastDisconnectTime_whenOnDeviceDisconnect_thenSkipsThisEvent(long negativeLastDisconnectTime) { // GIVEN doReturn(false).when(service).cleanDeviceStateIfBelongsToExternalPartition(tenantId, deviceId); @@ -254,7 +286,7 @@ public class DefaultDeviceStateServiceTest { @ParameterizedTest @MethodSource("provideOutdatedTimestamps") - public void givenOutdatedLastDisconnectTime_whenOnDeviceDisconnect_thenSkipsThisEvent(long outdatedLastDisconnectTime, long currentLastDisconnectTime) { + void givenOutdatedLastDisconnectTime_whenOnDeviceDisconnect_thenSkipsThisEvent(long outdatedLastDisconnectTime, long currentLastDisconnectTime) { // GIVEN doReturn(false).when(service).cleanDeviceStateIfBelongsToExternalPartition(tenantId, deviceId); @@ -275,7 +307,7 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceBelongsToMyPartition_whenOnDeviceDisconnect_thenReportsDisconnect() { + void givenDeviceBelongsToMyPartition_whenOnDeviceDisconnect_thenReportsDisconnect() { // GIVEN var deviceStateData = DeviceStateData.builder() .tenantId(tenantId) @@ -289,11 +321,13 @@ public class DefaultDeviceStateServiceTest { service.deviceStates.put(deviceId, deviceStateData); long lastDisconnectTime = System.currentTimeMillis(); + mockSuccessfulSaveAttributes(); + // WHEN service.onDeviceDisconnect(tenantId, deviceId, lastDisconnectTime); // THEN - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(LAST_DISCONNECT_TIME) && @@ -308,7 +342,7 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceBelongsToExternalPartition_whenOnDeviceInactivity_thenCleansStateAndDoesNotReportInactivity() { + void givenDeviceBelongsToExternalPartition_whenOnDeviceInactivity_thenCleansStateAndDoesNotReportInactivity() { // GIVEN doReturn(true).when(service).cleanDeviceStateIfBelongsToExternalPartition(tenantId, deviceId); @@ -325,7 +359,7 @@ public class DefaultDeviceStateServiceTest { @ParameterizedTest @ValueSource(longs = {Long.MIN_VALUE, -100, -1}) - public void givenNegativeLastInactivityTime_whenOnDeviceInactivity_thenSkipsThisEvent(long negativeLastInactivityTime) { + void givenNegativeLastInactivityTime_whenOnDeviceInactivity_thenSkipsThisEvent(long negativeLastInactivityTime) { // GIVEN doReturn(false).when(service).cleanDeviceStateIfBelongsToExternalPartition(tenantId, deviceId); @@ -341,7 +375,7 @@ public class DefaultDeviceStateServiceTest { @ParameterizedTest @MethodSource("provideOutdatedTimestamps") - public void givenReceivedInactivityTimeIsLessThanOrEqualToCurrentInactivityTime_whenOnDeviceInactivity_thenSkipsThisEvent( + void givenReceivedInactivityTimeIsLessThanOrEqualToCurrentInactivityTime_whenOnDeviceInactivity_thenSkipsThisEvent( long outdatedLastInactivityTime, long currentLastInactivityTime ) { // GIVEN @@ -365,7 +399,7 @@ public class DefaultDeviceStateServiceTest { @ParameterizedTest @MethodSource("provideOutdatedTimestamps") - public void givenReceivedInactivityTimeIsLessThanOrEqualToCurrentActivityTime_whenOnDeviceInactivity_thenSkipsThisEvent( + void givenReceivedInactivityTimeIsLessThanOrEqualToCurrentActivityTime_whenOnDeviceInactivity_thenSkipsThisEvent( long outdatedLastInactivityTime, long currentLastActivityTime ) { // GIVEN @@ -398,7 +432,7 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceBelongsToMyPartition_whenOnDeviceInactivity_thenReportsInactivity() { + void givenDeviceBelongsToMyPartition_whenOnDeviceInactivity_thenReportsInactivity() { // GIVEN var deviceStateData = DeviceStateData.builder() .tenantId(tenantId) @@ -412,17 +446,19 @@ public class DefaultDeviceStateServiceTest { service.deviceStates.put(deviceId, deviceStateData); long lastInactivityTime = System.currentTimeMillis(); + mockSuccessfulSaveAttributes(); + // WHEN service.onDeviceInactivity(tenantId, deviceId, lastInactivityTime); // THEN - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) && request.getEntries().get(0).getValue().equals(lastInactivityTime) )); - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && @@ -445,7 +481,7 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenInactivityTimeoutReached_whenUpdateInactivityStateIfExpired_thenReportsInactivity() { + void givenInactivityTimeoutReached_whenUpdateInactivityStateIfExpired_thenReportsInactivity() { // GIVEN var deviceStateData = DeviceStateData.builder() .tenantId(tenantId) @@ -456,16 +492,18 @@ public class DefaultDeviceStateServiceTest { given(partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId)).willReturn(tpi); + mockSuccessfulSaveAttributes(); + // WHEN service.updateInactivityStateIfExpired(System.currentTimeMillis(), deviceId, deviceStateData); // THEN - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) )); - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && @@ -488,7 +526,7 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceIdFromDeviceStatesMap_whenGetOrFetchDeviceStateData_thenNoStackOverflow() { + void givenDeviceIdFromDeviceStatesMap_whenGetOrFetchDeviceStateData_thenNoStackOverflow() { service.deviceStates.put(deviceId, deviceStateDataMock); DeviceStateData deviceStateData = service.getOrFetchDeviceStateData(deviceId); assertThat(deviceStateData).isEqualTo(deviceStateDataMock); @@ -496,7 +534,7 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceIdWithoutDeviceStateInMap_whenGetOrFetchDeviceStateData_thenFetchDeviceStateData() { + void givenDeviceIdWithoutDeviceStateInMap_whenGetOrFetchDeviceStateData_thenFetchDeviceStateData() { service.deviceStates.clear(); willReturn(deviceStateDataMock).given(service).fetchDeviceStateDataUsingSeparateRequests(deviceId); DeviceStateData deviceStateData = service.getOrFetchDeviceStateData(deviceId); @@ -504,172 +542,18 @@ public class DefaultDeviceStateServiceTest { verify(service).fetchDeviceStateDataUsingSeparateRequests(deviceId); } - private void initStateService(long timeout) throws InterruptedException { - service.stop(); - reset(service, telemetrySubscriptionService); - service.setDefaultInactivityTimeoutMs(timeout); - service.init(); - when(partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId)).thenReturn(tpi); - when(entityQueryRepository.findEntityDataByQueryInternal(any())).thenReturn(new PageData<>()); - var deviceIdInfo = new DeviceIdInfo(tenantId.getId(), null, deviceId.getId()); - when(deviceService.findDeviceIdInfos(any())) - .thenReturn(new PageData<>(List.of(deviceIdInfo), 0, 1, false)); - PartitionChangeEvent event = new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of( - new QueueKey(ServiceType.TB_CORE), Collections.singleton(tpi) - ), Collections.emptyMap()); - service.onApplicationEvent(event); - Thread.sleep(100); - } - - @Test - public void increaseInactivityForInactiveDeviceTest() throws Exception { - final long defaultTimeout = 1; - initStateService(defaultTimeout); - DeviceState deviceState = DeviceState.builder().build(); - DeviceStateData deviceStateData = DeviceStateData.builder() - .tenantId(tenantId) - .deviceId(deviceId) - .state(deviceState) - .metaData(new TbMsgMetaData()) - .build(); - - service.deviceStates.put(deviceId, deviceStateData); - service.getPartitionedEntities(tpi).add(deviceId); - - service.onDeviceActivity(tenantId, deviceId, System.currentTimeMillis()); - activityVerify(true); - Thread.sleep(defaultTimeout); - service.checkStates(); - activityVerify(false); - - reset(telemetrySubscriptionService); - - long increase = 100; - long newTimeout = System.currentTimeMillis() - deviceState.getLastActivityTime() + increase; - - service.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, newTimeout); - activityVerify(true); - Thread.sleep(increase); - service.checkStates(); - activityVerify(false); - - reset(telemetrySubscriptionService); - - service.onDeviceActivity(tenantId, deviceId, System.currentTimeMillis()); - activityVerify(true); - Thread.sleep(newTimeout + 5); - service.checkStates(); - activityVerify(false); - } - - @Test - public void increaseInactivityForActiveDeviceTest() throws Exception { - final long defaultTimeout = 1000; - initStateService(defaultTimeout); - DeviceState deviceState = DeviceState.builder().build(); - DeviceStateData deviceStateData = DeviceStateData.builder() - .tenantId(tenantId) - .deviceId(deviceId) - .state(deviceState) - .metaData(new TbMsgMetaData()) - .build(); - - service.deviceStates.put(deviceId, deviceStateData); - service.getPartitionedEntities(tpi).add(deviceId); - - service.onDeviceActivity(tenantId, deviceId, System.currentTimeMillis()); - activityVerify(true); - - reset(telemetrySubscriptionService); - - long increase = 100; - long newTimeout = System.currentTimeMillis() - deviceState.getLastActivityTime() + increase; - - service.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, newTimeout); - verify(telemetrySubscriptionService, never()).saveAttributes(argThat(request -> - request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) - )); - Thread.sleep(defaultTimeout + increase); - service.checkStates(); - activityVerify(false); - - reset(telemetrySubscriptionService); - - service.onDeviceActivity(tenantId, deviceId, System.currentTimeMillis()); - activityVerify(true); - Thread.sleep(newTimeout); - service.checkStates(); - activityVerify(false); - } + @MethodSource + @ParameterizedTest + void testOnDeviceInactivityTimeoutUpdate(boolean initialActivityStatus, long newInactivityTimeout, boolean expectedActivityStatus) { + // GIVEN + doReturn(200L).when(service).getCurrentTimeMillis(); - @Test - public void increaseSmallInactivityForInactiveDeviceTest() throws Exception { - final long defaultTimeout = 1; - initStateService(defaultTimeout); - DeviceState deviceState = DeviceState.builder().build(); - DeviceStateData deviceStateData = DeviceStateData.builder() - .tenantId(tenantId) - .deviceId(deviceId) - .state(deviceState) - .metaData(new TbMsgMetaData()) + var deviceState = DeviceState.builder() + .active(initialActivityStatus) + .lastActivityTime(100L) .build(); - service.deviceStates.put(deviceId, deviceStateData); - service.getPartitionedEntities(tpi).add(deviceId); - - service.onDeviceActivity(tenantId, deviceId, System.currentTimeMillis()); - activityVerify(true); - Thread.sleep(defaultTimeout); - service.checkStates(); - activityVerify(false); - - reset(telemetrySubscriptionService); - - long newTimeout = 1; - Thread.sleep(newTimeout); - verify(telemetrySubscriptionService, never()).saveAttributes(argThat(request -> - request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) - )); - } - - @Test - public void decreaseInactivityForActiveDeviceTest() throws Exception { - final long defaultTimeout = 1000; - initStateService(defaultTimeout); - DeviceState deviceState = DeviceState.builder().build(); - DeviceStateData deviceStateData = DeviceStateData.builder() - .tenantId(tenantId) - .deviceId(deviceId) - .state(deviceState) - .metaData(new TbMsgMetaData()) - .build(); - - service.deviceStates.put(deviceId, deviceStateData); - service.getPartitionedEntities(tpi).add(deviceId); - - service.onDeviceActivity(tenantId, deviceId, System.currentTimeMillis()); - activityVerify(true); - - long newTimeout = 1; - Thread.sleep(newTimeout); - - service.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, newTimeout); - activityVerify(false); - reset(telemetrySubscriptionService); - - service.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, defaultTimeout); - activityVerify(true); - Thread.sleep(defaultTimeout); - service.checkStates(); - activityVerify(false); - } - - @Test - public void decreaseInactivityForInactiveDeviceTest() throws Exception { - final long defaultTimeout = 1000; - initStateService(defaultTimeout); - DeviceState deviceState = DeviceState.builder().build(); - DeviceStateData deviceStateData = DeviceStateData.builder() + var deviceStateData = DeviceStateData.builder() .tenantId(tenantId) .deviceId(deviceId) .state(deviceState) @@ -679,31 +563,44 @@ public class DefaultDeviceStateServiceTest { service.deviceStates.put(deviceId, deviceStateData); service.getPartitionedEntities(tpi).add(deviceId); - service.onDeviceActivity(tenantId, deviceId, System.currentTimeMillis()); - activityVerify(true); - Thread.sleep(defaultTimeout); - service.checkStates(); - activityVerify(false); - reset(telemetrySubscriptionService); + mockSuccessfulSaveAttributes(); - long newTimeout = 1; + // WHEN + service.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, newInactivityTimeout); - service.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, newTimeout); - verify(telemetrySubscriptionService, never()).saveAttributes(argThat(request -> - request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) - )); + // THEN + long expectedInactivityTimeout = newInactivityTimeout != 0 ? newInactivityTimeout : defaultInactivityTimeoutMs; + assertThat(deviceState.getInactivityTimeout()).isEqualTo(expectedInactivityTimeout); + + assertThat(deviceState.isActive()).isEqualTo(expectedActivityStatus); + if (initialActivityStatus != expectedActivityStatus) { + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> { + AttributeKvEntry entry = request.getEntries().get(0); + return request.getEntityId().equals(deviceId) && entry.getKey().equals(ACTIVITY_STATE) && entry.getValue().equals(expectedActivityStatus); + })); + } } - private void activityVerify(boolean isActive) { - verify(telemetrySubscriptionService).saveAttributes(argThat(request -> - request.getEntityId().equals(deviceId) && - request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && - request.getEntries().get(0).getValue().equals(isActive) - )); + // to simplify test, these arguments assume that the current time is 200 and the last activity time is 100 + private static Stream testOnDeviceInactivityTimeoutUpdate() { + return Stream.of( + Arguments.of(true, 1L, false), + Arguments.of(true, 50L, false), + Arguments.of(true, 99L, false), + Arguments.of(true, 100L, false), + Arguments.of(true, 101L, true), + Arguments.of(true, 0L, true), // should use default inactivity timeout of 10 minutes + Arguments.of(false, 1L, false), + Arguments.of(false, 50L, false), + Arguments.of(false, 99L, false), + Arguments.of(false, 100L, false), + Arguments.of(false, 101L, true), + Arguments.of(false, 0L, true) // should use default inactivity timeout of 10 minutes + ); } @Test - public void givenStateDataIsNull_whenUpdateActivityState_thenShouldCleanupDevice() { + void givenStateDataIsNull_whenUpdateActivityState_thenShouldCleanupDevice() { // GIVEN service.deviceStates.put(deviceId, deviceStateDataMock); @@ -719,7 +616,7 @@ public class DefaultDeviceStateServiceTest { @ParameterizedTest @MethodSource("provideParametersForUpdateActivityState") - public void givenTestParameters_whenUpdateActivityState_thenShouldBeInTheExpectedStateAndPerformExpectedActions( + void givenTestParameters_whenUpdateActivityState_thenShouldBeInTheExpectedStateAndPerformExpectedActions( boolean activityState, long previousActivityTime, long lastReportedActivity, long inactivityAlarmTime, long expectedInactivityAlarmTime, boolean shouldSetInactivityAlarmTimeToZero, boolean shouldUpdateActivityStateToActive @@ -739,13 +636,15 @@ public class DefaultDeviceStateServiceTest { .metaData(new TbMsgMetaData()) .build(); + mockSuccessfulSaveAttributes(); + // WHEN service.updateActivityState(deviceId, deviceStateData, lastReportedActivity); // THEN assertThat(deviceState.isActive()).isEqualTo(true); assertThat(deviceState.getLastActivityTime()).isEqualTo(lastReportedActivity); - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(LAST_ACTIVITY_TIME) && request.getEntries().get(0).getValue().equals(lastReportedActivity) @@ -753,7 +652,7 @@ public class DefaultDeviceStateServiceTest { assertThat(deviceState.getLastInactivityAlarmTime()).isEqualTo(expectedInactivityAlarmTime); if (shouldSetInactivityAlarmTimeToZero) { - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) && request.getEntries().get(0).getValue().equals(0L) @@ -761,7 +660,7 @@ public class DefaultDeviceStateServiceTest { } if (shouldUpdateActivityStateToActive) { - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getValue().equals(true) @@ -809,59 +708,8 @@ public class DefaultDeviceStateServiceTest { ); } - @ParameterizedTest - @MethodSource("provideParametersForDecreaseInactivityTimeout") - public void givenTestParameters_whenOnDeviceInactivityTimeout_thenShouldBeInTheExpectedStateAndPerformExpectedActions( - boolean activityState, long newInactivityTimeout, long timeIncrement, boolean expectedActivityState - ) throws Exception { - // GIVEN - long defaultInactivityTimeout = 10000; - initStateService(defaultInactivityTimeout); - - var currentTime = new AtomicLong(System.currentTimeMillis()); - - DeviceState deviceState = DeviceState.builder() - .active(activityState) - .lastActivityTime(currentTime.get()) - .inactivityTimeout(defaultInactivityTimeout) - .build(); - - DeviceStateData deviceStateData = DeviceStateData.builder() - .tenantId(tenantId) - .deviceId(deviceId) - .state(deviceState) - .metaData(new TbMsgMetaData()) - .build(); - - service.deviceStates.put(deviceId, deviceStateData); - service.getPartitionedEntities(tpi).add(deviceId); - - given(service.getCurrentTimeMillis()).willReturn(currentTime.addAndGet(timeIncrement)); - - // WHEN - service.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, newInactivityTimeout); - - // THEN - assertThat(deviceState.getInactivityTimeout()).isEqualTo(newInactivityTimeout); - assertThat(deviceState.isActive()).isEqualTo(expectedActivityState); - if (activityState && !expectedActivityState) { - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && - request.getEntries().get(0).getValue().equals(false) - )); - } - } - - private static Stream provideParametersForDecreaseInactivityTimeout() { - return Stream.of( - Arguments.of(true, 1, 0, true), - - Arguments.of(true, 1, 1, false) - ); - } - @Test - public void givenStateDataIsNull_whenUpdateInactivityTimeoutIfExpired_thenShouldCleanupDevice() { + void givenStateDataIsNull_whenUpdateInactivityTimeoutIfExpired_thenShouldCleanupDevice() { // GIVEN service.deviceStates.put(deviceId, deviceStateDataMock); @@ -875,7 +723,7 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenNotMyPartition_whenUpdateInactivityTimeoutIfExpired_thenShouldCleanupDevice() { + void givenNotMyPartition_whenUpdateInactivityTimeoutIfExpired_thenShouldCleanupDevice() { // GIVEN long currentTime = System.currentTimeMillis(); @@ -911,7 +759,7 @@ public class DefaultDeviceStateServiceTest { @ParameterizedTest @MethodSource("provideParametersForUpdateInactivityStateIfExpired") - public void givenTestParameters_whenUpdateInactivityStateIfExpired_thenShouldBeInTheExpectedStateAndPerformExpectedActions( + void givenTestParameters_whenUpdateInactivityStateIfExpired_thenShouldBeInTheExpectedStateAndPerformExpectedActions( boolean activityState, long ts, long lastActivityTime, long lastInactivityAlarmTime, long inactivityTimeout, long deviceCreationTime, boolean expectedActivityState, long expectedLastInactivityAlarmTime, boolean shouldUpdateActivityStateToInactive ) { @@ -933,6 +781,7 @@ public class DefaultDeviceStateServiceTest { if (shouldUpdateActivityStateToInactive) { given(partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId)).willReturn(tpi); + mockSuccessfulSaveAttributes(); } // WHEN @@ -943,7 +792,7 @@ public class DefaultDeviceStateServiceTest { assertThat(state.getLastInactivityAlarmTime()).isEqualTo(expectedLastInactivityAlarmTime); if (shouldUpdateActivityStateToInactive) { - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getValue().equals(false) )); @@ -961,7 +810,7 @@ public class DefaultDeviceStateServiceTest { assertThat(actualNotification.getDeviceId()).isEqualTo(deviceId); assertThat(actualNotification.isActive()).isFalse(); - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) && @@ -1033,7 +882,80 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenConcurrentAccess_whenGetOrFetchDeviceStateData_thenFetchDeviceStateDataInvokedOnce() { + void givenInactiveDevice_whenActivityStatusChangesToActiveButFailedToSaveUpdatedActivityStatus_thenShouldNotUpdateCache() { + // GIVEN + doReturn(200L).when(service).getCurrentTimeMillis(); + + var deviceState = DeviceState.builder() + .active(false) + .lastActivityTime(100L) + .inactivityTimeout(50L) + .build(); + + var deviceStateData = DeviceStateData.builder() + .tenantId(tenantId) + .deviceId(deviceId) + .state(deviceState) + .metaData(TbMsgMetaData.EMPTY) + .build(); + + service.deviceStates.put(deviceId, deviceStateData); + service.getPartitionedEntities(tpi).add(deviceId); + + when(telemetrySubscriptionService.saveAttributesInternal(any(AttributesSaveRequest.class))) + .thenAnswer(invocation -> { + AttributesSaveRequest request = invocation.getArgument(0); + AttributeKvEntry entry = request.getEntries().get(0); + return entry.getKey().equals(ACTIVITY_STATE) ? + Futures.immediateFailedFuture(new RuntimeException("failed to save")) : + Futures.immediateFuture(generateRandomVersions(1)); + }); + + // WHEN + service.onDeviceActivity(tenantId, deviceId, 220L); + + // THEN + assertThat(deviceState.isActive()).isFalse(); + } + + @Test + void givenActiveDevice_whenActivityStatusChangesToInactiveButFailedToSaveUpdatedActivityStatus_thenShouldNotUpdateCache() { + // GIVEN + var deviceState = DeviceState.builder() + .active(true) + .lastActivityTime(100L) + .inactivityTimeout(50L) + .build(); + + var deviceStateData = DeviceStateData.builder() + .tenantId(tenantId) + .deviceId(deviceId) + .state(deviceState) + .metaData(TbMsgMetaData.EMPTY) + .build(); + + service.deviceStates.put(deviceId, deviceStateData); + service.getPartitionedEntities(tpi).add(deviceId); + + when(telemetrySubscriptionService.saveAttributesInternal(any(AttributesSaveRequest.class))) + .thenAnswer(invocation -> { + AttributesSaveRequest request = invocation.getArgument(0); + AttributeKvEntry entry = request.getEntries().get(0); + return entry.getKey().equals(ACTIVITY_STATE) ? + Futures.immediateFailedFuture(new RuntimeException("failed to save")) : + Futures.immediateFuture(generateRandomVersions(1)); + }); + + // WHEN + doReturn(200L).when(service).getCurrentTimeMillis(); + service.checkStates(); + + // THEN + assertThat(deviceState.isActive()).isTrue(); + } + + @Test + void givenConcurrentAccess_whenGetOrFetchDeviceStateData_thenFetchDeviceStateDataInvokedOnce() { doAnswer(invocation -> { Thread.sleep(100); return deviceStateDataMock; @@ -1069,10 +991,8 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceAdded_whenOnQueueMsg_thenShouldCacheAndSaveActivityToFalse() throws InterruptedException { + void givenDeviceAdded_whenOnQueueMsg_thenShouldCacheAndSaveActivityToFalse() { // GIVEN - final long defaultTimeout = 1000; - initStateService(defaultTimeout); given(deviceService.findDeviceById(any(TenantId.class), any(DeviceId.class))).willReturn(new Device(deviceId)); given(attributesService.find(any(TenantId.class), any(EntityId.class), any(AttributeScope.class), anyCollection())).willReturn(Futures.immediateFuture(Collections.emptyList())); @@ -1086,13 +1006,15 @@ public class DefaultDeviceStateServiceTest { .setDeleted(false) .build(); + mockSuccessfulSaveAttributes(); + // WHEN service.onQueueMsg(proto, TbCallback.EMPTY); // THEN await().atMost(1, TimeUnit.SECONDS).untilAsserted(() -> { assertThat(service.deviceStates.get(deviceId).getState().isActive()).isEqualTo(false); - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getValue().equals(false) )); @@ -1100,14 +1022,12 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceActivityEventHappenedAfterAdded_whenOnDeviceActivity_thenShouldCacheAndSaveActivityToTrue() throws InterruptedException { + void givenDeviceActivityEventHappenedAfterAdded_whenOnDeviceActivity_thenShouldCacheAndSaveActivityToTrue() { // GIVEN - final long defaultTimeout = 1000; - initStateService(defaultTimeout); long currentTime = System.currentTimeMillis(); DeviceState deviceState = DeviceState.builder() .active(false) - .inactivityTimeout(service.getDefaultInactivityTimeoutInSec()) + .inactivityTimeout(defaultInactivityTimeoutMs) .build(); DeviceStateData stateData = DeviceStateData.builder() .tenantId(tenantId) @@ -1118,12 +1038,14 @@ public class DefaultDeviceStateServiceTest { .build(); service.deviceStates.put(deviceId, stateData); + mockSuccessfulSaveAttributes(); + // WHEN service.onDeviceActivity(tenantId, deviceId, currentTime); // THEN ArgumentCaptor attributeRequestCaptor = ArgumentCaptor.forClass(AttributesSaveRequest.class); - then(telemetrySubscriptionService).should(times(2)).saveAttributes(attributeRequestCaptor.capture()); + then(telemetrySubscriptionService).should(times(2)).saveAttributesInternal(attributeRequestCaptor.capture()); await().atMost(1, TimeUnit.SECONDS).untilAsserted(() -> { assertThat(service.deviceStates.get(deviceId).getState().isActive()).isEqualTo(true); @@ -1151,15 +1073,14 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceActivityEventHappenedBeforeAdded_whenOnQueueMsg_thenShouldSaveActivityStateUsingValueFromCache() throws InterruptedException { + void givenDeviceActivityEventHappenedBeforeAdded_whenOnQueueMsg_thenShouldSaveActivityStateUsingValueFromCache() { // GIVEN - final long defaultTimeout = 1000; - initStateService(defaultTimeout); given(deviceService.findDeviceById(any(TenantId.class), any(DeviceId.class))).willReturn(new Device(deviceId)); given(attributesService.find(any(TenantId.class), any(EntityId.class), any(AttributeScope.class), anyCollection())).willReturn(Futures.immediateFuture(Collections.emptyList())); long currentTime = System.currentTimeMillis(); - DeviceState deviceState = DeviceState.builder() + + var deviceState = DeviceState.builder() .active(true) .lastConnectTime(currentTime - 8000) .lastActivityTime(currentTime - 4000) @@ -1167,16 +1088,20 @@ public class DefaultDeviceStateServiceTest { .lastInactivityAlarmTime(0) .inactivityTimeout(3000) .build(); - DeviceStateData stateData = DeviceStateData.builder() + + var stateData = DeviceStateData.builder() .tenantId(tenantId) .deviceId(deviceId) .deviceCreationTime(currentTime - 10000) .state(deviceState) .build(); + service.deviceStates.put(deviceId, stateData); + mockSuccessfulSaveAttributes(); + // WHEN - TransportProtos.DeviceStateServiceMsgProto proto = TransportProtos.DeviceStateServiceMsgProto.newBuilder() + var proto = TransportProtos.DeviceStateServiceMsgProto.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) @@ -1190,11 +1115,25 @@ public class DefaultDeviceStateServiceTest { // THEN await().atMost(1, TimeUnit.SECONDS).untilAsserted(() -> { assertThat(service.deviceStates.get(deviceId).getState().isActive()).isEqualTo(true); - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getValue().equals(true) )); }); } + private void mockSuccessfulSaveAttributes() { + lenient().when(telemetrySubscriptionService.saveAttributesInternal(any())).thenAnswer(invocation -> { + AttributesSaveRequest request = invocation.getArgument(0); + return Futures.immediateFuture(generateRandomVersions(request.getEntries().size())); + }); + } + + private static List generateRandomVersions(int n) { + return ThreadLocalRandom.current() + .longs(n) + .boxed() + .toList(); + } + } From d917c72d5198f4223fcf52b1e0e0a0f8904d74e6 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Fri, 6 Jun 2025 18:58:14 +0300 Subject: [PATCH 06/56] Use a callback to log errors on failure for improved error reporting --- .../state/DefaultDeviceStateService.java | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index f46323f702..13745ae6ab 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -604,19 +604,27 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService { - stateData.getState().setActive(active); - pushRuleEngineMessage(stateData, active ? TbMsgType.ACTIVITY_EVENT : TbMsgType.INACTIVITY_EVENT); - TbMsgMetaData metaData = stateData.getMetaData(); - notificationRuleProcessor.process(DeviceActivityTrigger.builder() - .tenantId(tenantId) - .customerId(stateData.getCustomerId()) - .deviceId(deviceId) - .active(active) - .deviceName(metaData.getValue("deviceName")) - .deviceType(metaData.getValue("deviceType")) - .deviceLabel(metaData.getValue("deviceLabel")) - .build()); + Futures.addCallback(save(tenantId, deviceId, ACTIVITY_STATE, active), new FutureCallback<>() { + @Override + public void onSuccess(Void success) { + stateData.getState().setActive(active); + pushRuleEngineMessage(stateData, active ? TbMsgType.ACTIVITY_EVENT : TbMsgType.INACTIVITY_EVENT); + TbMsgMetaData metaData = stateData.getMetaData(); + notificationRuleProcessor.process(DeviceActivityTrigger.builder() + .tenantId(tenantId) + .customerId(stateData.getCustomerId()) + .deviceId(deviceId) + .active(active) + .deviceName(metaData.getValue("deviceName")) + .deviceType(metaData.getValue("deviceType")) + .deviceLabel(metaData.getValue("deviceLabel")) + .build()); + } + + @Override + public void onFailure(@NonNull Throwable t) { + log.error("[{}][{}] Failed to change device activity status to '{}'. Device state data: {}", tenantId, deviceId, active, stateData, t); + } }, deviceStateCallbackExecutor); } From baaa9f723501e94b3c09c4e2d9792178b98fa2fc Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Fri, 6 Jun 2025 20:25:00 +0300 Subject: [PATCH 07/56] Update last inactivity alarm time after a successful database save; improve tests --- .../state/DefaultDeviceStateService.java | 25 ++++++--- .../state/DefaultDeviceStateServiceTest.java | 51 +++++++++---------- 2 files changed, 43 insertions(+), 33 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 13745ae6ab..23fdc6b8c0 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -340,7 +340,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService() { + @Override + public void onSuccess(Void success) { + stateData.getState().setLastInactivityAlarmTime(ts); + onDeviceActivityStatusChange(false, stateData); + } + + @Override + public void onFailure(@NonNull Throwable t) { + log.error("[{}][{}] Failed to update device last inactivity alarm time to '{}'. Device state data: {}", tenantId, deviceId, ts, stateData, t); + } + }, deviceStateCallbackExecutor); } private static boolean isActive(long ts, DeviceState state) { diff --git a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java index cbf7363441..0fe29eef57 100644 --- a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java @@ -882,10 +882,8 @@ class DefaultDeviceStateServiceTest { } @Test - void givenInactiveDevice_whenActivityStatusChangesToActiveButFailedToSaveUpdatedActivityStatus_thenShouldNotUpdateCache() { + void givenInactiveDevice_whenActivityStatusChangesToActiveButFailedToSaveUpdatedActivityStatus_thenShouldNotUpdateCache2() { // GIVEN - doReturn(200L).when(service).getCurrentTimeMillis(); - var deviceState = DeviceState.builder() .active(false) .lastActivityTime(100L) @@ -902,20 +900,21 @@ class DefaultDeviceStateServiceTest { service.deviceStates.put(deviceId, deviceStateData); service.getPartitionedEntities(tpi).add(deviceId); - when(telemetrySubscriptionService.saveAttributesInternal(any(AttributesSaveRequest.class))) - .thenAnswer(invocation -> { - AttributesSaveRequest request = invocation.getArgument(0); - AttributeKvEntry entry = request.getEntries().get(0); - return entry.getKey().equals(ACTIVITY_STATE) ? - Futures.immediateFailedFuture(new RuntimeException("failed to save")) : - Futures.immediateFuture(generateRandomVersions(1)); - }); + // WHEN-THEN - // WHEN - service.onDeviceActivity(tenantId, deviceId, 220L); + // simulating short DB outage + given(telemetrySubscriptionService.saveAttributesInternal(any())).willReturn(Futures.immediateFailedFuture(new RuntimeException("failed to save"))); + doReturn(200L).when(service).getCurrentTimeMillis(); + service.onDeviceActivity(tenantId, deviceId, 180L); + assertThat(deviceState.isActive()).isFalse(); // still inactive - // THEN - assertThat(deviceState.isActive()).isFalse(); + // 10 millis pass... and new activity message it received + + // this time DB save is successful + when(telemetrySubscriptionService.saveAttributesInternal(any())).thenReturn(Futures.immediateFuture(generateRandomVersions(1))); + doReturn(210L).when(service).getCurrentTimeMillis(); + service.onDeviceActivity(tenantId, deviceId, 190L); + assertThat(deviceState.isActive()).isTrue(); } @Test @@ -937,21 +936,21 @@ class DefaultDeviceStateServiceTest { service.deviceStates.put(deviceId, deviceStateData); service.getPartitionedEntities(tpi).add(deviceId); - when(telemetrySubscriptionService.saveAttributesInternal(any(AttributesSaveRequest.class))) - .thenAnswer(invocation -> { - AttributesSaveRequest request = invocation.getArgument(0); - AttributeKvEntry entry = request.getEntries().get(0); - return entry.getKey().equals(ACTIVITY_STATE) ? - Futures.immediateFailedFuture(new RuntimeException("failed to save")) : - Futures.immediateFuture(generateRandomVersions(1)); - }); + // WHEN-THEN (assuming periodic activity states check is done every 100 millis) - // WHEN + // simulating short DB outage + given(telemetrySubscriptionService.saveAttributesInternal(any())).willReturn(Futures.immediateFailedFuture(new RuntimeException("failed to save"))); doReturn(200L).when(service).getCurrentTimeMillis(); service.checkStates(); + assertThat(deviceState.isActive()).isTrue(); // still active - // THEN - assertThat(deviceState.isActive()).isTrue(); + // waiting 100 millis... periodic activity states check is triggered again + + // this time DB save is successful + when(telemetrySubscriptionService.saveAttributesInternal(any())).thenReturn(Futures.immediateFuture(generateRandomVersions(1))); + doReturn(300L).when(service).getCurrentTimeMillis(); + service.checkStates(); + assertThat(deviceState.isActive()).isFalse(); } @Test From 9a82f6b8817cd7017748056a606a73160bc2e42f Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Mon, 9 Jun 2025 15:47:52 +0300 Subject: [PATCH 08/56] Return `AttributesSaveResult` instead of just `List` when saving attributes --- .../cf/CalculatedFieldQueueService.java | 3 +- .../DefaultCalculatedFieldQueueService.java | 12 +++--- .../device/DeviceProvisionServiceImpl.java | 12 +++--- .../service/edge/rpc/EdgeGrpcSession.java | 16 ++++---- .../DefaultSystemDataLoaderService.java | 9 +++-- .../DefaultTelemetrySubscriptionService.java | 8 ++-- .../telemetry/InternalTelemetryService.java | 5 +-- .../service/entitiy/EntityServiceTest.java | 25 ++++++------ .../state/DefaultDeviceStateServiceTest.java | 5 ++- ...faultTelemetrySubscriptionServiceTest.java | 34 +++++++++++------ .../dao/attributes/AttributesService.java | 5 ++- .../common/data/kv/AttributesSaveResult.java | 32 ++++++++++++++++ .../dao/attributes/BaseAttributesService.java | 27 +++++++------ .../attributes/CachedAttributesService.java | 38 +++++++++---------- 14 files changed, 140 insertions(+), 91 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/kv/AttributesSaveResult.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java index eb86220361..f9ec8087ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java @@ -21,6 +21,7 @@ import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.RuleEngineCalculatedFieldQueueService; import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import java.util.List; @@ -35,7 +36,7 @@ public interface CalculatedFieldQueueService extends RuleEngineCalculatedFieldQu */ void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback); - void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); + void pushRequestToQueue(AttributesSaveRequest request, AttributesSaveResult result, FutureCallback callback); void pushRequestToQueue(AttributesDeleteRequest request, List result, FutureCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 8289e4db42..c3185738f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; @@ -96,7 +97,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } @Override - public void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback) { + public void pushRequestToQueue(AttributesSaveRequest request, AttributesSaveResult result, FutureCallback callback) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), @@ -186,17 +187,18 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS return msg.build(); } - private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List versions) { + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, AttributesSaveResult result) { ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); + List entries = request.getEntries(); + List versions = result.versions(); + for (int i = 0; i < entries.size(); i++) { AttributeValueProto.Builder attrProtoBuilder = ProtoUtils.toProto(entries.get(i)).toBuilder(); - if (versions != null) { - attrProtoBuilder.setVersion(versions.get(i)); - } + attrProtoBuilder.setVersion(versions.get(i)); telemetryMsg.addAttrData(attrProtoBuilder.build()); } msg.setTelemetryMsg(telemetryMsg.build()); diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java index 173ba742af..0778d61ee7 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.msg.TbMsgType; @@ -62,8 +63,6 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; -import java.util.Collections; -import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.regex.Matcher; @@ -240,10 +239,11 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService { return deviceCredentialsService.updateDeviceCredentials(tenantId, deviceCredentials); } - private ListenableFuture> saveProvisionStateAttribute(Device device) { - return attributesService.save(device.getTenantId(), device.getId(), AttributeScope.SERVER_SCOPE, - Collections.singletonList(new BaseAttributeKvEntry(new StringDataEntry(DEVICE_PROVISION_STATE, PROVISIONED_STATE), - System.currentTimeMillis()))); + private ListenableFuture saveProvisionStateAttribute(Device device) { + return attributesService.save( + device.getTenantId(), device.getId(), AttributeScope.SERVER_SCOPE, + new BaseAttributeKvEntry(new StringDataEntry(DEVICE_PROVISION_STATE, PROVISIONED_STATE), System.currentTimeMillis()) + ); } private DeviceCredentials getDeviceCredentials(Device device) { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index 4a9b68fc6d..8922fa6008 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; @@ -42,7 +43,6 @@ import org.thingsboard.server.common.data.limit.LimitedApi; import org.thingsboard.server.common.data.notification.rule.trigger.EdgeCommunicationFailureTrigger; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; @@ -582,10 +582,10 @@ public abstract class EdgeGrpcSession implements Closeable { @Override public void onSuccess(@Nullable Pair newStartTsAndSeqId) { if (newStartTsAndSeqId != null) { - ListenableFuture> updateFuture = updateQueueStartTsAndSeqId(newStartTsAndSeqId); + ListenableFuture updateFuture = updateQueueStartTsAndSeqId(newStartTsAndSeqId); Futures.addCallback(updateFuture, new FutureCallback<>() { @Override - public void onSuccess(@Nullable List list) { + public void onSuccess(@Nullable AttributesSaveResult saveResult) { log.debug("[{}][{}] queue offset was updated [{}]", tenantId, sessionId, newStartTsAndSeqId); boolean newEventsAvailable; if (fetcher.isSeqIdNewCycleStarted()) { @@ -646,8 +646,7 @@ public abstract class EdgeGrpcSession implements Closeable { log.trace("[{}][{}] entity message processed [{}]", tenantId, sessionId, downlinkMsg); } } - case ATTRIBUTES_UPDATED, POST_ATTRIBUTES, ATTRIBUTES_DELETED, TIMESERIES_UPDATED -> - downlinkMsg = ctx.getTelemetryProcessor().convertTelemetryEventToDownlink(edge, edgeEvent); + case ATTRIBUTES_UPDATED, POST_ATTRIBUTES, ATTRIBUTES_DELETED, TIMESERIES_UPDATED -> downlinkMsg = ctx.getTelemetryProcessor().convertTelemetryEventToDownlink(edge, edgeEvent); default -> log.warn("[{}][{}] Unsupported action type [{}]", tenantId, sessionId, edgeEvent.getAction()); } } catch (Exception e) { @@ -723,13 +722,14 @@ public abstract class EdgeGrpcSession implements Closeable { return startSeqId; } - private ListenableFuture> updateQueueStartTsAndSeqId(Pair pair) { + private ListenableFuture updateQueueStartTsAndSeqId(Pair pair) { newStartTs = pair.getFirst(); newStartSeqId = pair.getSecond(); log.trace("[{}] updateQueueStartTsAndSeqId [{}][{}][{}]", sessionId, edge.getId(), newStartTs, newStartSeqId); - List attributes = Arrays.asList( + List attributes = List.of( new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_TS_ATTR_KEY, newStartTs), System.currentTimeMillis()), - new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_SEQ_ID_ATTR_KEY, newStartSeqId), System.currentTimeMillis())); + new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_SEQ_ID_ATTR_KEY, newStartSeqId), System.currentTimeMillis()) + ); return ctx.getAttributesService().save(edge.getTenantId(), edge.getId(), AttributeScope.SERVER_SCOPE, attributes); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index e9ef8c5ace..d580175aa0 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -64,6 +64,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; @@ -581,9 +582,9 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { Collections.singletonList(new BasicTsKvEntry(System.currentTimeMillis(), new BooleanDataEntry(key, value))), 0L); addTsCallback(saveFuture, new TelemetrySaveCallback<>(deviceId, key, value)); } else { - ListenableFuture> saveFuture = attributesService.save(TenantId.SYS_TENANT_ID, deviceId, AttributeScope.SERVER_SCOPE, - Collections.singletonList(new BaseAttributeKvEntry(new BooleanDataEntry(key, value) - , System.currentTimeMillis()))); + ListenableFuture saveFuture = attributesService.save( + TenantId.SYS_TENANT_ID, deviceId, AttributeScope.SERVER_SCOPE, new BaseAttributeKvEntry(new BooleanDataEntry(key, value), System.currentTimeMillis()) + ); addTsCallback(saveFuture, new TelemetrySaveCallback<>(deviceId, key, value)); } } @@ -611,7 +612,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { } private void addTsCallback(ListenableFuture saveFuture, final FutureCallback callback) { - Futures.addCallback(saveFuture, new FutureCallback() { + Futures.addCallback(saveFuture, new FutureCallback<>() { @Override public void onSuccess(@Nullable S result) { callback.onSuccess(result); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 4640b9339f..0ff2d42f15 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -45,6 +45,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -62,7 +63,6 @@ import org.thingsboard.server.service.state.DefaultDeviceStateService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -190,16 +190,16 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } @Override - public ListenableFuture> saveAttributesInternal(AttributesSaveRequest request) { + public ListenableFuture saveAttributesInternal(AttributesSaveRequest request) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); AttributesSaveRequest.Strategy strategy = request.getStrategy(); - ListenableFuture> resultFuture; + ListenableFuture resultFuture; if (strategy.saveAttributes()) { resultFuture = attrService.save(tenantId, entityId, request.getScope(), request.getEntries()); } else { - resultFuture = Futures.immediateFuture(Collections.emptyList()); + resultFuture = Futures.immediateFuture(AttributesSaveResult.EMPTY); } addMainCallback(resultFuture, result -> { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java index 79f0beab41..2d3be5a0ba 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java @@ -21,10 +21,9 @@ import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; -import java.util.List; - /** * Created by ashvayka on 27.03.18. */ @@ -32,7 +31,7 @@ public interface InternalTelemetryService extends RuleEngineTelemetryService { ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request); - ListenableFuture> saveAttributesInternal(AttributesSaveRequest request); + ListenableFuture saveAttributesInternal(AttributesSaveRequest request); void deleteTimeseriesInternal(TimeseriesDeleteRequest request); diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index 264ac70443..d8bcd42bc8 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.IdBased; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; @@ -395,7 +396,7 @@ public class EntityServiceTest extends AbstractControllerTest { List highTemperatures = new ArrayList<>(); createTestHierarchy(tenantId, assets, devices, new ArrayList<>(), new ArrayList<>(), temperatures, highTemperatures); - List>> attributeFutures = new ArrayList<>(); + List> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), AttributeScope.CLIENT_SCOPE)); @@ -545,7 +546,7 @@ public class EntityServiceTest extends AbstractControllerTest { List highTemperatures = new ArrayList<>(); createTestHierarchy(tenantId, assets, devices, new ArrayList<>(), new ArrayList<>(), temperatures, highTemperatures); - List>> attributeFutures = new ArrayList<>(); + List> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), AttributeScope.CLIENT_SCOPE)); @@ -599,7 +600,7 @@ public class EntityServiceTest extends AbstractControllerTest { List highConsumptions = new ArrayList<>(); createTestHierarchy(tenantId, assets, devices, consumptions, highConsumptions, new ArrayList<>(), new ArrayList<>()); - List>> attributeFutures = new ArrayList<>(); + List> attributeFutures = new ArrayList<>(); for (int i = 0; i < assets.size(); i++) { Asset asset = assets.get(i); attributeFutures.add(saveLongAttribute(asset.getId(), "consumption", consumptions.get(i), AttributeScope.SERVER_SCOPE)); @@ -1506,7 +1507,7 @@ public class EntityServiceTest extends AbstractControllerTest { } } - List>> attributeFutures = new ArrayList<>(); + List> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); for (AttributeScope currentScope : AttributeScope.values()) { @@ -1578,7 +1579,7 @@ public class EntityServiceTest extends AbstractControllerTest { } } - List>> attributeFutures = new ArrayList<>(); + List> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), AttributeScope.CLIENT_SCOPE)); @@ -1808,7 +1809,7 @@ public class EntityServiceTest extends AbstractControllerTest { } } - List>> attributeFutures = new ArrayList<>(); + List> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); attributeFutures.add(saveStringAttribute(device.getId(), "attributeString", attributeStrings.get(i), AttributeScope.CLIENT_SCOPE)); @@ -2269,16 +2270,16 @@ public class EntityServiceTest extends AbstractControllerTest { return filter; } - private ListenableFuture> saveLongAttribute(EntityId entityId, String key, long value, AttributeScope scope) { + private ListenableFuture saveLongAttribute(EntityId entityId, String key, long value, AttributeScope scope) { KvEntry attrValue = new LongDataEntry(key, value); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); - return attributesService.save(tenantId, entityId, scope, Collections.singletonList(attr)); + return attributesService.save(tenantId, entityId, scope, List.of(attr)); } - private ListenableFuture> saveStringAttribute(EntityId entityId, String key, String value, AttributeScope scope) { + private ListenableFuture saveStringAttribute(EntityId entityId, String key, String value, AttributeScope scope) { KvEntry attrValue = new StringDataEntry(key, value); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); - return attributesService.save(tenantId, entityId, scope, Collections.singletonList(attr)); + return attributesService.save(tenantId, entityId, scope, List.of(attr)); } private ListenableFuture saveTimeseries(EntityId entityId, String key, Double value) { @@ -2294,8 +2295,8 @@ public class EntityServiceTest extends AbstractControllerTest { } protected void createMultiRootHierarchy(List buildings, List apartments, - Map> entityNameByTypeMap, - Map childParentRelationMap) throws InterruptedException { + Map> entityNameByTypeMap, + Map childParentRelationMap) throws InterruptedException { for (int k = 0; k < 3; k++) { Asset building = new Asset(); building.setTenantId(tenantId); diff --git a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java index 0fe29eef57..5bf87137ff 100644 --- a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.notification.rule.trigger.DeviceActivityTrigger; import org.thingsboard.server.common.msg.TbMsg; @@ -911,7 +912,7 @@ class DefaultDeviceStateServiceTest { // 10 millis pass... and new activity message it received // this time DB save is successful - when(telemetrySubscriptionService.saveAttributesInternal(any())).thenReturn(Futures.immediateFuture(generateRandomVersions(1))); + when(telemetrySubscriptionService.saveAttributesInternal(any())).thenReturn(Futures.immediateFuture(AttributesSaveResult.of(generateRandomVersions(1)))); doReturn(210L).when(service).getCurrentTimeMillis(); service.onDeviceActivity(tenantId, deviceId, 190L); assertThat(deviceState.isActive()).isTrue(); @@ -947,7 +948,7 @@ class DefaultDeviceStateServiceTest { // waiting 100 millis... periodic activity states check is triggered again // this time DB save is successful - when(telemetrySubscriptionService.saveAttributesInternal(any())).thenReturn(Futures.immediateFuture(generateRandomVersions(1))); + when(telemetrySubscriptionService.saveAttributesInternal(any())).thenReturn(Futures.immediateFuture(AttributesSaveResult.of(generateRandomVersions(1)))); doReturn(300L).when(service).getCurrentTimeMillis(); service.checkStates(); assertThat(deviceState.isActive()).isFalse(); diff --git a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java index 2b4d9f38e5..153228a865 100644 --- a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java @@ -48,6 +48,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; @@ -472,7 +473,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(saveAttributes, sendWsUpdate, processCalculatedFields)) .build(); - lenient().when(attrService.save(tenantId, entityId, request.getScope(), request.getEntries())).thenReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + lenient().when(attrService.save(tenantId, entityId, request.getScope(), request.getEntries())) + .thenReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(request.getEntries().size())))); // WHEN telemetryService.saveAttributes(request); @@ -547,7 +549,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, deviceId, request.getScope(), entries)).willReturn(immediateFuture(listOfNNumbers(entries.size()))); + given(attrService.save(tenantId, deviceId, request.getScope(), entries)) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(entries.size())))); // WHEN telemetryService.saveAttributes(request); @@ -581,7 +584,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, nonDeviceId, request.getScope(), entries)).willReturn(immediateFuture(listOfNNumbers(entries.size()))); + given(attrService.save(tenantId, nonDeviceId, request.getScope(), entries)) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(entries.size())))); // WHEN telemetryService.saveAttributes(request); @@ -613,7 +617,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, deviceId, request.getScope(), entries)).willReturn(immediateFuture(listOfNNumbers(entries.size()))); + given(attrService.save(tenantId, deviceId, request.getScope(), entries)) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(entries.size())))); // WHEN telemetryService.saveAttributes(request); @@ -640,7 +645,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, deviceId, request.getScope(), entries)).willReturn(immediateFuture(listOfNNumbers(entries.size()))); + given(attrService.save(tenantId, deviceId, request.getScope(), entries)) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(entries.size())))); // WHEN telemetryService.saveAttributes(request); @@ -715,7 +721,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())).willReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(request.getEntries().size())))); // WHEN telemetryService.saveAttributes(request); @@ -764,7 +771,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, nonDeviceId, request.getScope(), request.getEntries())).willReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + given(attrService.save(tenantId, nonDeviceId, request.getScope(), request.getEntries())) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(request.getEntries().size())))); // WHEN telemetryService.saveAttributes(request); @@ -792,7 +800,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())).willReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(request.getEntries().size())))); // WHEN telemetryService.saveAttributes(request); @@ -815,7 +824,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())).willReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(request.getEntries().size())))); // WHEN telemetryService.saveAttributes(request); @@ -843,7 +853,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())).willReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(request.getEntries().size())))); // WHEN telemetryService.saveAttributes(request); @@ -870,7 +881,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())).willReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(request.getEntries().size())))); // WHEN telemetryService.saveAttributes(request); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java index 718c574c3c..0d5d3dcd13 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import java.util.Collection; import java.util.List; @@ -37,9 +38,9 @@ public interface AttributesService { ListenableFuture> findAll(TenantId tenantId, EntityId entityId, AttributeScope scope); - ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes); + ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes); - ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute); + ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute); ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributeKeys); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/AttributesSaveResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/AttributesSaveResult.java new file mode 100644 index 0000000000..711a3e06a4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/AttributesSaveResult.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.kv; + +import java.util.Collections; +import java.util.List; + +public record AttributesSaveResult(List versions) { + + public static final AttributesSaveResult EMPTY = new AttributesSaveResult(Collections.emptyList()); + + public static AttributesSaveResult of(List versions) { + if (versions == null) { + return EMPTY; + } + return new AttributesSaveResult(versions); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 777a77d054..9803670d4b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.dao.service.Validator; @@ -41,7 +42,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; import static org.thingsboard.server.dao.attributes.AttributeUtils.validate; @@ -101,26 +101,29 @@ public class BaseAttributesService implements AttributesService { } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { validate(entityId, scope); AttributeUtils.validate(attribute, valueNoXssValidation); - return doSave(tenantId, entityId, scope, attribute); + return doSave(tenantId, entityId, scope, List.of(attribute)); } @Override - public ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { validate(entityId, scope); AttributeUtils.validate(attributes, valueNoXssValidation); - List> saveFutures = attributes.stream().map(attribute -> doSave(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); - return Futures.allAsList(saveFutures); + return doSave(tenantId, entityId, scope, attributes); } - private ListenableFuture doSave(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { - ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); - return Futures.transform(future, version -> { - edqsService.onUpdate(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attribute, version)); - return version; - }, MoreExecutors.directExecutor()); + private ListenableFuture doSave(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { + List> futures = new ArrayList<>(attributes.size()); + for (AttributeKvEntry attribute : attributes) { + ListenableFuture future = Futures.transform(attributesDao.save(tenantId, entityId, scope, attribute), version -> { + edqsService.onUpdate(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attribute, version)); + return version; + }, MoreExecutors.directExecutor()); + futures.add(future); + } + return Futures.transform(Futures.allAsList(futures), AttributesSaveResult::of, MoreExecutors.directExecutor()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 559828911f..d99413f13a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.msg.edqs.EdqsService; @@ -56,7 +57,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; import static org.thingsboard.server.dao.attributes.AttributeUtils.validate; @@ -150,7 +150,7 @@ public class CachedAttributesService implements AttributesService { List cachedAttributes = wrappedCachedAttributes.values().stream() .map(TbCacheValueWrapper::get) .filter(Objects::nonNull) - .collect(Collectors.toList()); + .toList(); if (wrappedCachedAttributes.size() == attributeKeys.size()) { log.trace("[{}][{}] Found all attributes from cache: {}", entityId, scope, attributeKeys); return Futures.immediateFuture(cachedAttributes); @@ -159,8 +159,6 @@ public class CachedAttributesService implements AttributesService { Set notFoundAttributeKeys = new HashSet<>(attributeKeys); notFoundAttributeKeys.removeAll(wrappedCachedAttributes.keySet()); - List notFoundKeys = notFoundAttributeKeys.stream().map(k -> new AttributeCacheKey(scope, entityId, k)).collect(Collectors.toList()); - // DB call should run in DB executor, not in cache-related executor return jpaExecutorService.submit(() -> { log.trace("[{}][{}] Lookup attributes from db: {}", entityId, scope, notFoundAttributeKeys); @@ -222,33 +220,31 @@ public class CachedAttributesService implements AttributesService { } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { validate(entityId, scope); AttributeUtils.validate(attribute, valueNoXssValidation); - return doSave(tenantId, entityId, scope, attribute); + return doSave(tenantId, entityId, scope, List.of(attribute)); } @Override - public ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { validate(entityId, scope); AttributeUtils.validate(attributes, valueNoXssValidation); + return doSave(tenantId, entityId, scope, attributes); + } + private ListenableFuture doSave(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { List> futures = new ArrayList<>(attributes.size()); for (var attribute : attributes) { - futures.add(doSave(tenantId, entityId, scope, attribute)); + ListenableFuture future = Futures.transform(attributesDao.save(tenantId, entityId, scope, attribute), version -> { + BaseAttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(((BaseAttributeKvEntry) attribute).getKv(), attribute.getLastUpdateTs(), version); + put(entityId, scope, attributeKvEntry); + edqsService.onUpdate(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attributeKvEntry, version)); + return version; + }, cacheExecutor); + futures.add(future); } - - return Futures.allAsList(futures); - } - - private ListenableFuture doSave(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { - ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); - return Futures.transform(future, version -> { - BaseAttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(((BaseAttributeKvEntry) attribute).getKv(), attribute.getLastUpdateTs(), version); - put(entityId, scope, attributeKvEntry); - edqsService.onUpdate(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attributeKvEntry, version)); - return version; - }, cacheExecutor); + return Futures.transform(Futures.allAsList(futures), AttributesSaveResult::of, MoreExecutors.directExecutor()); } private void put(EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { @@ -270,7 +266,7 @@ public class CachedAttributesService implements AttributesService { edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); } return key; - }, cacheExecutor)).collect(Collectors.toList())); + }, cacheExecutor)).toList()); } @Override From da90d8f727c567fc535ed7cd983c618e17a5a39d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 10 Jun 2025 09:24:49 +0300 Subject: [PATCH 09/56] changed msgTs to latestTs --- .../controller/CalculatedFieldController.java | 4 +- .../DefaultCalculatedFieldQueueService.java | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 8 ++- .../cf/ctx/state/CalculatedFieldState.java | 2 +- .../ctx/state/ScriptCalculatedFieldState.java | 2 +- .../ctx/state/SimpleCalculatedFieldState.java | 6 +- .../cf/CalculatedFieldIntegrationTest.java | 64 ++++++++++++++++++- .../script/api/tbel/TbelCfCtx.java | 6 +- .../calculated-field-dialog.component.scss | 2 +- .../shared/models/calculated-field.models.ts | 8 +-- .../en_US/calculated-field/expression_fn.md | 8 +-- .../assets/locale/locale.constant-en_US.json | 4 +- 12 files changed, 90 insertions(+), 26 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 3257b31ca4..2dcb32cf39 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -244,7 +244,7 @@ public class CalculatedFieldController extends BaseController { ); Object[] args = new Object[ctxAndArgNames.size()]; - args[0] = new TbelCfCtx(arguments, getLastUpdateTimestamp(arguments)); + args[0] = new TbelCfCtx(arguments, getLatestTimestamp(arguments)); for (int i = 1; i < ctxAndArgNames.size(); i++) { var arg = arguments.get(ctxAndArgNames.get(i)); if (arg instanceof TbelCfSingleValueArg svArg) { @@ -267,7 +267,7 @@ public class CalculatedFieldController extends BaseController { return result; } - private long getLastUpdateTimestamp(Map arguments) { + private long getLatestTimestamp(Map arguments) { long lastUpdateTimestamp = -1; for (TbelCfArg entry : arguments.values()) { if (entry instanceof TbelCfSingleValueArg singleValueArg) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 8289e4db42..81f19e1d1a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -176,7 +176,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS for (int i = 0; i < entries.size(); i++) { TsKvProto.Builder tsProtoBuilder = toTsKvProto(entries.get(i)).toBuilder(); - if (result != null) { + if (versions != null && !versions.isEmpty() && versions.get(i) != null) { tsProtoBuilder.setVersion(versions.get(i)); } telemetryMsg.addTsData(tsProtoBuilder.build()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index e4b03b4cab..e21d56b6d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -35,7 +35,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected Map arguments; protected boolean sizeExceedsLimit; - protected long lastUpdateTimestamp = -1; + protected long latestTimestamp = -1; public BaseCalculatedFieldState(List requiredArguments) { this.requiredArguments = requiredArguments; @@ -110,12 +110,14 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected abstract void validateNewEntry(ArgumentEntry newEntry); private void updateLastUpdateTimestamp(ArgumentEntry entry) { + long newTs = this.latestTimestamp; if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - this.lastUpdateTimestamp = singleValueArgumentEntry.getTs(); + newTs = singleValueArgumentEntry.getTs(); } else if (entry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { Map.Entry lastEntry = tsRollingArgumentEntry.getTsRecords().lastEntry(); - this.lastUpdateTimestamp = (lastEntry != null) ? lastEntry.getKey() : System.currentTimeMillis(); + newTs = (lastEntry != null) ? lastEntry.getKey() : System.currentTimeMillis(); } + this.latestTimestamp = Math.max(this.latestTimestamp, newTs); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 6eac3358ba..0de354bbb0 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -42,7 +42,7 @@ public interface CalculatedFieldState { Map getArguments(); - long getLastUpdateTimestamp(); + long getLatestTimestamp(); void setRequiredArguments(List requiredArguments); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 65ef40330c..84dce627ae 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -66,7 +66,7 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { args.add(arg); } } - args.set(0, new TbelCfCtx(arguments, getLastUpdateTimestamp())); + args.set(0, new TbelCfCtx(arguments, getLatestTimestamp())); ListenableFuture resultFuture = ctx.getCalculatedFieldScriptEngine().executeJsonAsync(args.toArray()); Output output = ctx.getOutput(); return Futures.transform(resultFuture, diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index d0eba5031c..c8cdc7b4c0 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -87,10 +87,10 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { ObjectNode valuesNode = JacksonUtil.newObjectNode(); valuesNode.set(outputName, JacksonUtil.valueToTree(result)); - long lastTimestamp = getLastUpdateTimestamp(); - if (preserveMsgTs && lastTimestamp != -1) { + long latestTs = getLatestTimestamp(); + if (preserveMsgTs && latestTs != -1) { ObjectNode resultNode = JacksonUtil.newObjectNode(); - resultNode.put("ts", lastTimestamp); + resultNode.put("ts", latestTs); resultNode.set("values", valuesNode); return resultNode; } else { diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index f65f6bc629..9742c7f618 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -505,6 +505,68 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes }); } + @Test + public void testSimpleCalculatedFieldWhenPreserveMsgTsIsTrueAndTelemetryBeforeLatest() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + long ts = System.currentTimeMillis(); + + long tsA = ts - 300000L; + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"a\":1}}", tsA))); + + long tsB = ts - 300L; + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"b\":5}}", tsB))); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("a + b"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument1 = new Argument(); + ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("a", ArgumentType.TS_LATEST, null); + argument1.setRefEntityKey(refEntityKey1); + Argument argument2 = new Argument(); + ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("b", ArgumentType.TS_LATEST, null); + argument2.setRefEntityKey(refEntityKey2); + config.setArguments(Map.of("a", argument1, "b", argument2)); + config.setExpression("a + b"); + + Output output = new Output(); + output.setName("c"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + config.setPreserveMsgTs(true); + + calculatedField.setConfiguration(config); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + await().alias("create CF -> perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode c = getLatestTelemetry(testDevice.getId(), "c"); + assertThat(c).isNotNull(); + assertThat(c.get("c").get(0).get("ts").asText()).isEqualTo(Long.toString(tsB)); + assertThat(c.get("c").get(0).get("value").asText()).isEqualTo("6.0"); + }); + + long tsABeforeTsB = tsB - 300L; + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"b\":10}}", tsABeforeTsB))); + + await().alias("update telemetry with ts less than latest -> save result with latest ts").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode c = getLatestTelemetry(testDevice.getId(), "c"); + assertThat(c).isNotNull(); + assertThat(c.get("c").get(0).get("ts").asText()).isEqualTo(Long.toString(tsB));// also tsB, since this is the latest timestamp + assertThat(c.get("c").get(0).get("value").asText()).isEqualTo("11.0"); + }); + } + @Test public void testScriptCalculatedFieldWhenUsedMsgTsInScript() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); @@ -524,7 +586,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); - config.setExpression("return {\"ts\": ctx.msgTs, \"values\": {\"fahrenheitTemp\": (T * 1.8) + 32}};"); + config.setExpression("return {\"ts\": ctx.latestTs, \"values\": {\"fahrenheitTemp\": (T * 1.8) + 32}};"); Output output = new Output(); output.setType(OutputType.TIME_SERIES); diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfCtx.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfCtx.java index 7515cb5269..c6023154ea 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfCtx.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfCtx.java @@ -25,11 +25,11 @@ public class TbelCfCtx implements TbelCfObject { @Getter private final Map args; @Getter - private final long msgTs; + private final long latestTs; - public TbelCfCtx(Map args, long lastUpdateTs) { + public TbelCfCtx(Map args, long latestTs) { this.args = Collections.unmodifiableMap(args); - this.msgTs = lastUpdateTs != -1 ? lastUpdateTs : System.currentTimeMillis(); + this.latestTs = latestTs != -1 ? latestTs : System.currentTimeMillis(); } @Override diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss index efcd62efd4..e192e3ccc0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss @@ -45,7 +45,7 @@ &-key { color: #c24c1a; } - &-time-window, &-values, &-func, &-value, &-ts, &-msgTs { + &-time-window, &-values, &-func, &-value, &-ts, &-latestTs { color: #7214D0; } &-start-ts, &-end-ts { diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 6fade1d193..76b775b2fd 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -526,10 +526,10 @@ export const getCalculatedFieldArgumentsEditorCompleter = (argumentsObj: Record< description: 'Calculated field context arguments.', children: {} }, - msgTs: { + latestTs: { meta: 'constant', type: 'number', - description: 'Timestamp (ms) of the telemetry message that triggered the calculated field execution.' + description: 'Latest timestamp (ms) of the arguments telemetry.' } } } @@ -582,8 +582,8 @@ const calculatedFieldArgumentsContextValueHighlightRules: AceHighlightRules = { next: 'calculatedFieldCtxArgs' }, { - token: 'tb.calculated-field-msgTs', - regex: /msgTs/, + token: 'tb.calculated-field-latestTs', + regex: /latestTs/, next: 'no_regex' }, endGroupHighlightRule diff --git a/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md b/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md index 4c54b01499..1d45dea3c8 100644 --- a/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md +++ b/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md @@ -1,7 +1,7 @@ ## Calculated Field TBEL Script Function The **calculate()** function is a user-defined script that enables custom calculations using [TBEL](${siteBaseUrl}/docs${docPlatformPrefix}/user-guide/tbel/) on telemetry and attribute data. -It receives arguments configured in the calculated field setup, along with an additional `ctx` object that stores `msgTs` and provides access to all arguments. +It receives arguments configured in the calculated field setup, along with an additional `ctx` object that stores `latestTs` and provides access to all arguments. ### Function Signature @@ -216,14 +216,14 @@ The return format depends on the output type configured in the calculated field ### Message timestamp -The `ctx` object also includes property `msgTs`, which represents the timestamp of the incoming telemetry message that triggered the calculated field execution in milliseconds. +The `ctx` object also includes property `latestTs`, which represents the latest timestamp of the arguments telemetry in milliseconds. -You can use `ctx.msgTs` to set the timestamp of the resulting output explicitly when returning a time series object. +You can use `ctx.latestTs` to set the timestamp of the resulting output explicitly when returning a time series object. ```javascript var temperatureC = (temperatureF - 32) / 1.8; return { - ts: ctx.msgTs, + ts: ctx.latestTs, values: { "temperatureC": toFixed(temperatureC, 2) } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 5afc040daf..a0bb9c6052 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1069,7 +1069,7 @@ "delete-multiple-title": "Are you sure you want to delete { count, plural, =1 {1 calculated field} other {# calculated fields} }?", "delete-multiple-text": "Be careful, after the confirmation all selected calculated fields will be removed and all related data will become unrecoverable.", "test-with-this-message": "Test with this message", - "use-message-timestamp": "Use message timestamp", + "use-message-timestamp": "Use latest timestamp", "hint": { "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with time series rolling type.", "arguments-empty": "Arguments should not be empty.", @@ -1086,7 +1086,7 @@ "decimals-range": "Decimals by default should be a number between 0 and 15.", "expression": "Default expression demonstrates how to transform a temperature from Fahrenheit to Celsius.", "arguments-entity-not-found": "Argument target entity not found.", - "use-message-timestamp": "If enabled, the calculated value will be persisted using the timestamp of the telemetry that triggered the calculation, instead of the server time." + "use-message-timestamp": "If enabled, the calculated value will be persisted using the most recent timestamp from the arguments telemetry, instead of the server time." } }, "confirm-on-exit": { From cce9b8f24d948f34a4bd6f3ed7ffd14ff8147bc1 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 10 Jun 2025 19:19:38 +0300 Subject: [PATCH 10/56] EdgeGrpcService - destroy if previous session exists --- .../server/service/edge/rpc/EdgeGrpcService.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java index 5c95a0e98f..5671ffb2ab 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java @@ -335,6 +335,9 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i Edge edge = edgeGrpcSession.getEdge(); TenantId tenantId = edge.getTenantId(); log.info("[{}][{}] edge [{}] connected successfully.", tenantId, edgeGrpcSession.getSessionId(), edgeId); + if (sessions.containsKey(edgeId)) { + destroySession(sessions.get(edgeId)); + } sessions.put(edgeId, edgeGrpcSession); final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock()); newEventLock.lock(); @@ -503,7 +506,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } finally { newEventLock.unlock(); } - toRemove.destroy(); + destroySession(toRemove); TenantId tenantId = toRemove.getEdge().getTenantId(); save(tenantId, edgeId, ACTIVITY_STATE, false); long lastDisconnectTs = System.currentTimeMillis(); @@ -516,6 +519,12 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i edgeIdServiceIdCache.evict(edgeId); } + private void destroySession(EdgeGrpcSession session) { + try (session) { + session.destroy(); + } + } + private void save(TenantId tenantId, EdgeId edgeId, String key, long value) { log.debug("[{}][{}] Updating long edge telemetry [{}] [{}]", tenantId, edgeId, key, value); if (persistToTelemetry) { From 1692c9b92ab1f843db7e41becc94574b07a73683 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 10 Jun 2025 19:22:00 +0300 Subject: [PATCH 11/56] EDQS - fixed relations query in case multiple previous path are present --- .../query/processor/AbstractRelationQueryProcessor.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java index 2842d57ff0..193f3bb22e 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.edqs.query.processor; +import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; import org.thingsboard.server.common.data.permission.QueryContext; import org.thingsboard.server.common.data.query.EntityFilter; @@ -106,7 +107,7 @@ public abstract class AbstractRelationQueryProcessor ext private Set> getEntitiesSet(RelationsRepo relations) { Set> result = new HashSet<>(); - Set processed = new HashSet<>(); + Set processed = new HashSet<>(); Queue tasks = new LinkedList<>(); int maxLvl = getMaxLevel() == 0 ? MAXIMUM_QUERY_LEVEL : Math.max(1, getMaxLevel()); for (UUID uuid : getRootEntities()) { @@ -114,7 +115,7 @@ public abstract class AbstractRelationQueryProcessor ext } while (!tasks.isEmpty()) { RelationSearchTask task = tasks.poll(); - if (processed.add(task.entityId)) { + if (processed.add(task)) { var entityLvl = task.lvl + 1; Set entities = EntitySearchDirection.FROM.equals(getDirection()) ? relations.getFrom(task.entityId) : relations.getTo(task.entityId); if (isFetchLastLevelOnly() && entities.isEmpty() && task.previous != null && check(task.previous)) { @@ -157,6 +158,7 @@ public abstract class AbstractRelationQueryProcessor ext protected abstract boolean check(RelationInfo relationInfo); @RequiredArgsConstructor + @EqualsAndHashCode private static class RelationSearchTask { private final UUID entityId; private final int lvl; From e21850f5713547da7ebd86733d61cea7936a7008 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 10 Jun 2025 19:29:16 +0300 Subject: [PATCH 12/56] EDQS - added SQL like style for filter contains/starts with/ends with --- ...stractEntityProfileNameQueryProcessor.java | 2 +- .../AbstractEntityProfileQueryProcessor.java | 2 +- .../processor/EntityNameQueryProcessor.java | 2 +- .../server/edqs/util/RepositoryUtils.java | 60 +++++++++++++------ .../server/edqs/repo/RepositoryUtilsTest.java | 40 ++++++++++++- 5 files changed, 84 insertions(+), 22 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java index b78e49879e..f881d616ef 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java @@ -36,7 +36,7 @@ public abstract class AbstractEntityProfileNameQueryProcessor(getProfileNames(this.filter)); - pattern = RepositoryUtils.toSqlLikePattern(getEntityNameFilter(filter)); + pattern = RepositoryUtils.toContainsSqlLikePattern(getEntityNameFilter(filter)); } protected abstract String getEntityNameFilter(T filter); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java index 301ead7c63..9d043ff6fc 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java @@ -43,7 +43,7 @@ public abstract class AbstractEntityProfileQueryProcessor value.equals(predicateValue); - case STARTS_WITH -> value.startsWith(predicateValue); - case ENDS_WITH -> value.endsWith(predicateValue); + case STARTS_WITH -> toStartsWithSqlLikePattern(predicateValue).matcher(value).matches(); + case ENDS_WITH -> toEndsWithSqlLikePattern(predicateValue).matcher(value).matches(); case NOT_EQUAL -> !value.equals(predicateValue); - case CONTAINS -> value.contains(predicateValue); - case NOT_CONTAINS -> !value.contains(predicateValue); + case CONTAINS -> toContainsSqlLikePattern(predicateValue).matcher(value).matches(); + case NOT_CONTAINS -> !toContainsSqlLikePattern(predicateValue).matcher(value).matches(); case IN -> equalsAny(value, splitByCommaWithoutQuotes(predicateValue)); case NOT_IN -> !equalsAny(value, splitByCommaWithoutQuotes(predicateValue)); }; @@ -304,6 +304,15 @@ public class RepositoryUtils { return true; } else if (filterPredicates.getOperation() == OR) { for (KeyFilterPredicate filterPredicate : filterPredicates.getPredicates()) { + + // Emulate the SQL-like behavior of ThingsBoard's Entity Data Query service: + // for COMPLEX filters, return no results if filter value is empty + if (filterPredicate instanceof StringFilterPredicate stringFilterPredicate) { + if (StringUtils.isEmpty(stringFilterPredicate.getValue().getValue())) { + continue; + } + } + if (simpleKeyFilter.check(value, filterPredicate)) { return true; } @@ -314,23 +323,38 @@ public class RepositoryUtils { } } - public static Pattern toSqlLikePattern(String nameFilter) { - if (StringUtils.isNotBlank(nameFilter)) { - boolean percentSymbolOnStart = nameFilter.startsWith("%"); - boolean percentSymbolOnEnd = nameFilter.endsWith("%"); - if (percentSymbolOnStart) { - nameFilter = nameFilter.substring(1); - } - if (percentSymbolOnEnd) { - nameFilter = nameFilter.substring(0, nameFilter.length() - 1); - } - if (percentSymbolOnStart || percentSymbolOnEnd) { - return Pattern.compile((percentSymbolOnStart ? ".*" : "") + Pattern.quote(nameFilter) + (percentSymbolOnEnd ? ".*" : ""), Pattern.CASE_INSENSITIVE); + public static Pattern toContainsSqlLikePattern(String filter) { + if (StringUtils.isNotBlank(filter)) { + return toSqlLikePattern(filter, ".*", ".*"); + } + return null; + } + + private static Pattern toStartsWithSqlLikePattern(String filter) { + return toSqlLikePattern(filter, "^", ".*"); + } + + private static Pattern toEndsWithSqlLikePattern(String filter) { + return toSqlLikePattern(filter, ".*", "$"); + } + + private static Pattern toSqlLikePattern(String value, String prefix, String suffix ) { + if (value.contains("%") || value.contains("_")) { + String regexValue = value + .replace("_", ".") + .replace("%", ".*"); + String regex; + if ("^".equals(prefix)) { + regex = "^" + regexValue + (regexValue.endsWith(".*") ? "" : ".*"); + } else if ("$".equals(suffix)) { + regex = (regexValue.startsWith(".*") ? "" : ".*") + regexValue + "$"; } else { - return Pattern.compile(Pattern.quote(nameFilter) + ".*", Pattern.CASE_INSENSITIVE); + regex = (regexValue.startsWith(".*") ? "" : ".*") + regexValue + (regexValue.endsWith(".*") ? "" : ".*"); } + return Pattern.compile(regex, Pattern.CASE_INSENSITIVE); + } else { + return Pattern.compile(prefix + Pattern.quote(value) + suffix, Pattern.CASE_INSENSITIVE); } - return null; } @FunctionalInterface diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java index 6c7444c92a..fa3784ca19 100644 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java @@ -70,7 +70,45 @@ public class RepositoryUtilsTest { Arguments.of("loranet 123", getNameFilter(StringOperation.IN, "loranet 123, loranet 124"), true), Arguments.of("loranet 123", getNameFilter(StringOperation.IN, "loranet 125, loranet 126"), false), Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 125, loranet 126"), true), - Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 123, loranet 126"), false) + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 123, loranet 126"), false), + + // Basic CONTAINS + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%loranet"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet%"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%ranet%"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%123"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%loranx%"), false), + + // Basic STARTS_WITH + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "loranet%"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "lora%"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "lorax%"), false), + + // Basic ENDS_WITH + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "%123"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "%23"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "%124"), false), + + // CONTAINS with _ + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet_123"), true), // '_' = ' ' + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet_12_"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loran_t%"), true), + + // STARTS_WITH with _ + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "loranet_"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "lora__t%"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "lor_net%"), true), + + // ENDS_WITH with _ + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "_23"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "_2_"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "_3"), true), + + // Mixed patterns + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "lora__t 1%"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "lora%net%3"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%o_anet%2_3"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "lora___ ___"), true) ); } From 58aa930d1ddf9a9fcd1525171a207f04ba4a9ff2 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 11 Jun 2025 08:17:40 +0300 Subject: [PATCH 13/56] renamed property --- .../service/cf/ctx/state/CalculatedFieldCtx.java | 4 ++-- .../cf/ctx/state/SimpleCalculatedFieldState.java | 6 +++--- .../server/cf/CalculatedFieldIntegrationTest.java | 10 +++++----- .../SimpleCalculatedFieldConfiguration.java | 2 +- .../dialog/calculated-field-dialog.component.html | 2 +- .../dialog/calculated-field-dialog.component.ts | 14 +++++++------- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index a3fdae319d..dff715a6e8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -62,7 +62,7 @@ public class CalculatedFieldCtx { private final List argNames; private Output output; private String expression; - private boolean preserveMsgTs; + private boolean useLatestTs; private TbelInvokeService tbelInvokeService; private CalculatedFieldScriptEngine calculatedFieldScriptEngine; private ThreadLocal customExpression; @@ -96,7 +96,7 @@ public class CalculatedFieldCtx { this.argNames = new ArrayList<>(arguments.keySet()); this.output = configuration.getOutput(); this.expression = configuration.getExpression(); - this.preserveMsgTs = CalculatedFieldType.SIMPLE.equals(calculatedField.getType()) && ((SimpleCalculatedFieldConfiguration) configuration).isPreserveMsgTs(); + this.useLatestTs = CalculatedFieldType.SIMPLE.equals(calculatedField.getType()) && ((SimpleCalculatedFieldConfiguration) configuration).isUseLatestTs(); this.tbelInvokeService = tbelInvokeService; this.maxDataPointsPerRollingArg = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index c8cdc7b4c0..111624882b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -68,7 +68,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { Output output = ctx.getOutput(); Object result = formatResult(expressionResult, output.getDecimalsByDefault()); - JsonNode outputResult = createResultJson(ctx.isPreserveMsgTs(), output.getName(), result); + JsonNode outputResult = createResultJson(ctx.isUseLatestTs(), output.getName(), result); return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), outputResult)); } @@ -83,12 +83,12 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { return TbUtils.toFixed(expressionResult, decimals); } - private JsonNode createResultJson(boolean preserveMsgTs, String outputName, Object result) { + private JsonNode createResultJson(boolean useLatestTs, String outputName, Object result) { ObjectNode valuesNode = JacksonUtil.newObjectNode(); valuesNode.set(outputName, JacksonUtil.valueToTree(result)); long latestTs = getLatestTimestamp(); - if (preserveMsgTs && latestTs != -1) { + if (useLatestTs && latestTs != -1) { ObjectNode resultNode = JacksonUtil.newObjectNode(); resultNode.put("ts", latestTs); resultNode.set("values", valuesNode); diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index 9742c7f618..8214a03616 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -464,7 +464,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes } @Test - public void testSimpleCalculatedFieldWhenPreserveMsgTsIsTrue() throws Exception { + public void testSimpleCalculatedFieldWhenUseLatestTsIsTrue() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); long ts = System.currentTimeMillis() - 300000L; doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"temperature\":30}}", ts))); @@ -489,7 +489,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes output.setType(OutputType.TIME_SERIES); config.setOutput(output); - config.setPreserveMsgTs(true); + config.setUseLatestTs(true); calculatedField.setConfiguration(config); @@ -506,7 +506,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes } @Test - public void testSimpleCalculatedFieldWhenPreserveMsgTsIsTrueAndTelemetryBeforeLatest() throws Exception { + public void testSimpleCalculatedFieldWhenUseLatestTsIsTrueAndTelemetryBeforeLatest() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); long ts = System.currentTimeMillis(); @@ -539,7 +539,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes output.setType(OutputType.TIME_SERIES); config.setOutput(output); - config.setPreserveMsgTs(true); + config.setUseLatestTs(true); calculatedField.setConfiguration(config); @@ -568,7 +568,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes } @Test - public void testScriptCalculatedFieldWhenUsedMsgTsInScript() throws Exception { + public void testScriptCalculatedFieldWhenUsedLatestTsInScript() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); long ts = System.currentTimeMillis() - 300000L; doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"temperature\":30}}", ts))); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java index af3cb4d5cd..86c7b9e9b6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldType; @EqualsAndHashCode(callSuper = true) public class SimpleCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { - private boolean preserveMsgTs; + private boolean useLatestTs; @Override public CalculatedFieldType getType() { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 60f33ff005..47463e8da6 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -190,7 +190,7 @@
- +
calculated-fields.use-message-timestamp
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 4aa4eca425..975744b2c6 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -77,7 +77,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent Date: Wed, 11 Jun 2025 11:28:25 +0300 Subject: [PATCH 14/56] renamed keys --- .../components/dialog/calculated-field-dialog.component.html | 4 ++-- ui-ngx/src/assets/locale/locale.constant-en_US.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 47463e8da6..7b69d26a60 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -191,8 +191,8 @@
-
- calculated-fields.use-message-timestamp +
+ calculated-fields.use-latest-timestamp
diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 990979d405..a98b2aecf9 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1069,7 +1069,7 @@ "delete-multiple-title": "Are you sure you want to delete { count, plural, =1 {1 calculated field} other {# calculated fields} }?", "delete-multiple-text": "Be careful, after the confirmation all selected calculated fields will be removed and all related data will become unrecoverable.", "test-with-this-message": "Test with this message", - "use-message-timestamp": "Use latest timestamp", + "use-latest-timestamp": "Use latest timestamp", "hint": { "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with time series rolling type.", "arguments-empty": "Arguments should not be empty.", @@ -1086,7 +1086,7 @@ "decimals-range": "Decimals by default should be a number between 0 and 15.", "expression": "Default expression demonstrates how to transform a temperature from Fahrenheit to Celsius.", "arguments-entity-not-found": "Argument target entity not found.", - "use-message-timestamp": "If enabled, the calculated value will be persisted using the most recent timestamp from the arguments telemetry, instead of the server time." + "use-latest-timestamp": "If enabled, the calculated value will be persisted using the most recent timestamp from the arguments telemetry, instead of the server time." } }, "confirm-on-exit": { From aaead182634cb583dfcd79ad11d808d1bb9ee1c7 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 11 Jun 2025 12:16:56 +0300 Subject: [PATCH 15/56] Fixed case sensitive matcher --- .../org/thingsboard/server/edqs/util/RepositoryUtils.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java index 1a47c4814f..1550c1df30 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java @@ -54,7 +54,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -338,7 +337,7 @@ public class RepositoryUtils { return toSqlLikePattern(filter, ".*", "$"); } - private static Pattern toSqlLikePattern(String value, String prefix, String suffix ) { + private static Pattern toSqlLikePattern(String value, String prefix, String suffix) { if (value.contains("%") || value.contains("_")) { String regexValue = value .replace("_", ".") @@ -351,9 +350,9 @@ public class RepositoryUtils { } else { regex = (regexValue.startsWith(".*") ? "" : ".*") + regexValue + (regexValue.endsWith(".*") ? "" : ".*"); } - return Pattern.compile(regex, Pattern.CASE_INSENSITIVE); + return Pattern.compile(regex); } else { - return Pattern.compile(prefix + Pattern.quote(value) + suffix, Pattern.CASE_INSENSITIVE); + return Pattern.compile(prefix + Pattern.quote(value) + suffix); } } From cba1972109c173d1c29797b943ed48b9223bf40e Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 11 Jun 2025 14:43:22 +0300 Subject: [PATCH 16/56] fixed null pointer exception --- .../service/sync/ie/importing/impl/ResourceImportService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java index 96fcdd5425..37639f6a90 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java @@ -66,7 +66,7 @@ public class ResourceImportService extends BaseEntityImportService Date: Wed, 11 Jun 2025 16:27:03 +0300 Subject: [PATCH 17/56] trigger processing using telemetry sub service in CF node --- .../DefaultCalculatedFieldQueueService.java | 2 +- .../ctx/state/SimpleCalculatedFieldState.java | 21 +++++++++++++++++-- .../DefaultTelemetrySubscriptionService.java | 7 ++++--- .../src/main/resources/thingsboard.yml | 1 + .../engine/api/AttributesSaveRequest.java | 1 + .../engine/api/TimeseriesSaveRequest.java | 1 + .../telemetry/TbCalculatedFieldsNode.java | 6 ++++-- 7 files changed, 31 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index bf206bc5dc..0c2d1a24a9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -200,7 +200,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS for (int i = 0; i < entries.size(); i++) { TsKvEntry tsKvEntry = entries.get(i); - if (result != null) { + if (versions != null && !versions.isEmpty() && versions.get(i) != null) { tsKvEntry.setVersion(versions.get(i)); } telemetryMsg.addTsData(toTsKvProto(tsKvEntry)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index d0eba5031c..03399348cb 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -58,7 +58,18 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { for (Map.Entry entry : this.arguments.entrySet()) { try { BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getKvEntryValue(); - expr.setVariable(entry.getKey(), Double.parseDouble(kvEntry.getValueAsString())); + try { + double value = switch (kvEntry.getDataType()) { + case LONG -> kvEntry.getLongValue().map(Long::doubleValue).orElseThrow(); + case DOUBLE -> kvEntry.getDoubleValue().orElseThrow(); + case BOOLEAN -> kvEntry.getBooleanValue().map(b -> b ? 1.0 : 0.0).orElseThrow(); + case STRING -> Double.parseDouble(kvEntry.getValueAsString()); + case JSON -> Double.parseDouble(kvEntry.getValueAsString()); + }; + expr.setVariable(entry.getKey(), value); + } catch (Exception e) { + throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number.", e); + } } catch (NumberFormatException e) { throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); } @@ -85,7 +96,13 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { private JsonNode createResultJson(boolean preserveMsgTs, String outputName, Object result) { ObjectNode valuesNode = JacksonUtil.newObjectNode(); - valuesNode.set(outputName, JacksonUtil.valueToTree(result)); + if (result instanceof Double doubleValue) { + valuesNode.put(outputName, doubleValue); + } else if (result instanceof Integer integerValue) { + valuesNode.put(outputName, integerValue); + } else { + valuesNode.set(outputName, JacksonUtil.valueToTree(result)); + } long lastTimestamp = getLastUpdateTimestamp(); if (preserveMsgTs && lastTimestamp != -1) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 30f2885e78..0f4f615af3 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -28,7 +28,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.DonAsynchron; -import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.rule.engine.api.AttributesDeleteRequest; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.DeviceStateManager; @@ -69,7 +69,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.function.Consumer; import static java.util.Comparator.comparing; @@ -96,6 +95,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Value("${sql.ts.value_no_xss_validation:false}") private boolean valueNoXssValidation; + @Value("${sql.ts.thread_pool_size:12}") + private int threadPoolSize; public DefaultTelemetrySubscriptionService(AttributesService attrService, TimeseriesService tsService, @@ -116,7 +117,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @PostConstruct public void initExecutor() { super.initExecutor(); - tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ts-callback")); + tsCallBackExecutor = ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "ts-service-ts-callback"); } @Override diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 6d27b9bd7e..e170695b90 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -346,6 +346,7 @@ sql: stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}" # Interval in milliseconds for printing timeseries insert statistic batch_threads: "${SQL_TS_BATCH_THREADS:3}" # batch thread count has to be a prime number like 3 or 5 to gain perfect hash distribution value_no_xss_validation: "${SQL_TS_VALUE_NO_XSS_VALIDATION:false}" # If true telemetry values will be checked for XSS vulnerability + thread_pool_size: "${SQL_TS_THREAD_POOL_SIZE:12}"# Thread pool size to execute dynamic queries ts_latest: batch_size: "${SQL_TS_LATEST_BATCH_SIZE:1000}" # Batch size for persisting latest telemetry updates batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:50}" # Maximum timeout for latest telemetry entries queue polling. The value set in milliseconds diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java index c3095836bd..19c1956463 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java @@ -57,6 +57,7 @@ public class AttributesSaveRequest implements CalculatedFieldSystemAwareRequest public static final Strategy PROCESS_ALL = new Strategy(true, true, true); public static final Strategy WS_ONLY = new Strategy(false, true, false); public static final Strategy SKIP_ALL = new Strategy(false, false, false); + public static final Strategy CF_ONLY = new Strategy(false, false, true); } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index c402a0c984..3bacf8d8c3 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -56,6 +56,7 @@ public class TimeseriesSaveRequest implements CalculatedFieldSystemAwareRequest public static final Strategy WS_ONLY = new Strategy(false, false, true, false); public static final Strategy LATEST_AND_WS = new Strategy(false, true, true, false); public static final Strategy SKIP_ALL = new Strategy(false, false, false, false); + public static final Strategy CF_ONLY = new Strategy(false, false, false, true); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java index b2cce52c87..4e319500d1 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java @@ -92,13 +92,14 @@ public class TbCalculatedFieldsNode implements TbNode { .customerId(msg.getCustomerId()) .entityId(msg.getOriginator()) .entries(tsKvEntryList) + .strategy(TimeseriesSaveRequest.Strategy.CF_ONLY) .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) .tbMsgId(msg.getId()) .tbMsgType(msg.getInternalType()) .callback(new TelemetryNodeCallback(ctx, msg)) .build(); - ctx.getCalculatedFieldQueueService().pushRequestToQueue(timeseriesSaveRequest, timeseriesSaveRequest.getCallback()); + ctx.getTelemetryService().saveTimeseries(timeseriesSaveRequest); } private void processPostAttributesRequest(TbContext ctx, TbMsg msg) { @@ -114,12 +115,13 @@ public class TbCalculatedFieldsNode implements TbNode { .entityId(msg.getOriginator()) .scope(AttributeScope.valueOf(msg.getMetaData().getValue(SCOPE))) .entries(newAttributes) + .strategy(AttributesSaveRequest.Strategy.CF_ONLY) .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) .tbMsgId(msg.getId()) .tbMsgType(msg.getInternalType()) .callback(new TelemetryNodeCallback(ctx, msg)) .build(); - ctx.getCalculatedFieldQueueService().pushRequestToQueue(attributesSaveRequest, attributesSaveRequest.getCallback()); + ctx.getTelemetryService().saveAttributes(attributesSaveRequest); } } From d8a7f973975abf1f002e4025bdf9d5101eb70e7b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 11 Jun 2025 16:38:44 +0300 Subject: [PATCH 18/56] fixed updating entry with old telemetry --- .../service/cf/ctx/state/SingleValueArgumentEntry.java | 2 +- .../service/cf/ctx/state/SingleValueArgumentEntryTest.java | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 3b858e81b2..1585c9b2a9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -107,7 +107,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { @Override public boolean updateEntry(ArgumentEntry entry) { if (entry instanceof SingleValueArgumentEntry singleValueEntry) { - if (singleValueEntry.getTs() == this.ts) { + if (singleValueEntry.getTs() <= this.ts) { return false; } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java index 50cac8a6fe..5ea3808468 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -53,7 +53,7 @@ public class SingleValueArgumentEntryTest { } @Test - void testUpdateEntryWithThaSameTs() { + void testUpdateEntryWithTheSameTs() { assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts, new LongDataEntry("key", 13L), 363L))).isFalse(); } @@ -81,6 +81,11 @@ public class SingleValueArgumentEntryTest { assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 11L), 364L))).isTrue(); } + @Test + void testUpdateEntryWithOldTs() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts - 10, new LongDataEntry("key", 14L), 365L))).isFalse(); + } + @Test void testToTbelCfArgWhenJsonIsObject() { entry = new SingleValueArgumentEntry(ts, new JsonDataEntry("key", "{\"test\": 10}"), 370L); From 1042ab8d16956ded6615691ec8aabd9a1a27a674 Mon Sep 17 00:00:00 2001 From: Ekaterina Chantsova Date: Wed, 11 Jun 2025 19:29:59 +0300 Subject: [PATCH 19/56] UI: fixed updating timewindow configuration --- ui-ngx/src/app/shared/models/time/time.models.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/shared/models/time/time.models.ts b/ui-ngx/src/app/shared/models/time/time.models.ts index acb62d9d2a..c204cb6867 100644 --- a/ui-ngx/src/app/shared/models/time/time.models.ts +++ b/ui-ngx/src/app/shared/models/time/time.models.ts @@ -431,7 +431,7 @@ export const initModelFromDefaultTimewindow = (value: Timewindow, quickIntervalO if (isDefined(value.history.quickInterval)) { model.history.quickInterval = value.history.quickInterval; } - if (isDefined(value.history.fixedTimewindow)) { + if (isDefinedAndNotNull(value.history.fixedTimewindow)) { if (isDefined(value.history.fixedTimewindow.startTimeMs)) { model.history.fixedTimewindow.startTimeMs = value.history.fixedTimewindow.startTimeMs; } From 866f03cd7139a6eed9f818750bfee3cd85129e72 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 12 Jun 2025 07:48:29 +0300 Subject: [PATCH 20/56] fixed test --- .../thingsboard/server/cf/CalculatedFieldIntegrationTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index 8214a03616..c8b8b0244b 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -555,7 +555,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes }); long tsABeforeTsB = tsB - 300L; - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"b\":10}}", tsABeforeTsB))); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"a\":10}}", tsABeforeTsB))); await().alias("update telemetry with ts less than latest -> save result with latest ts").atMost(TIMEOUT, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) @@ -563,7 +563,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes ObjectNode c = getLatestTelemetry(testDevice.getId(), "c"); assertThat(c).isNotNull(); assertThat(c.get("c").get(0).get("ts").asText()).isEqualTo(Long.toString(tsB));// also tsB, since this is the latest timestamp - assertThat(c.get("c").get(0).get("value").asText()).isEqualTo("11.0"); + assertThat(c.get("c").get(0).get("value").asText()).isEqualTo("15.0"); }); } From 41117aff2d531517380dbc7cef2c634cb7fd1514 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 12 Jun 2025 12:26:30 +0300 Subject: [PATCH 21/56] removed redudant try-catch --- ...CalculatedFieldEntityMessageProcessor.java | 3 +-- ...faultCalculatedFieldProcessingService.java | 4 +--- .../ctx/state/SimpleCalculatedFieldState.java | 19 +++++++------------ .../src/main/resources/thingsboard.yml | 2 +- 4 files changed, 10 insertions(+), 18 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 5ea79e42a8..ebc4e60709 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -19,7 +19,6 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.DebugModeUtil; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; @@ -299,7 +298,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM callback.onSuccess(); } if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { - systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, calculationResult.getResult().toString(), null); } } } else { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index e9a6cb09aa..f2a6916751 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -25,7 +25,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.math.NumberUtils; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; @@ -66,7 +65,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNot import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; @@ -171,7 +169,7 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP OutputType type = calculatedFieldResult.getType(); TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; - TbMsg msg = TbMsg.newMsg().type(msgType).originator(entityId).previousCalculatedFieldIds(cfIds).metaData(md).data(JacksonUtil.writeValueAsString(calculatedFieldResult.getResult())).build(); + TbMsg msg = TbMsg.newMsg().type(msgType).originator(entityId).previousCalculatedFieldIds(cfIds).metaData(md).data(calculatedFieldResult.getResult().toString()).build(); clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index c07923e470..577ff80219 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -58,18 +58,13 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { for (Map.Entry entry : this.arguments.entrySet()) { try { BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getKvEntryValue(); - try { - double value = switch (kvEntry.getDataType()) { - case LONG -> kvEntry.getLongValue().map(Long::doubleValue).orElseThrow(); - case DOUBLE -> kvEntry.getDoubleValue().orElseThrow(); - case BOOLEAN -> kvEntry.getBooleanValue().map(b -> b ? 1.0 : 0.0).orElseThrow(); - case STRING -> Double.parseDouble(kvEntry.getValueAsString()); - case JSON -> Double.parseDouble(kvEntry.getValueAsString()); - }; - expr.setVariable(entry.getKey(), value); - } catch (Exception e) { - throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number.", e); - } + double value = switch (kvEntry.getDataType()) { + case LONG -> kvEntry.getLongValue().map(Long::doubleValue).orElseThrow(); + case DOUBLE -> kvEntry.getDoubleValue().orElseThrow(); + case BOOLEAN -> kvEntry.getBooleanValue().map(b -> b ? 1.0 : 0.0).orElseThrow(); + case STRING, JSON -> Double.parseDouble(kvEntry.getValueAsString()); + }; + expr.setVariable(entry.getKey(), value); } catch (NumberFormatException e) { throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index c04969bfc7..958ad2f84c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -346,7 +346,7 @@ sql: stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}" # Interval in milliseconds for printing timeseries insert statistic batch_threads: "${SQL_TS_BATCH_THREADS:3}" # batch thread count has to be a prime number like 3 or 5 to gain perfect hash distribution value_no_xss_validation: "${SQL_TS_VALUE_NO_XSS_VALIDATION:false}" # If true telemetry values will be checked for XSS vulnerability - thread_pool_size: "${SQL_TS_THREAD_POOL_SIZE:12}"# Thread pool size to execute dynamic queries + thread_pool_size: "${SQL_TS_THREAD_POOL_SIZE:12}" # Thread pool size for telemetry callback executor ts_latest: batch_size: "${SQL_TS_LATEST_BATCH_SIZE:1000}" # Batch size for persisting latest telemetry updates batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:50}" # Maximum timeout for latest telemetry entries queue polling. The value set in milliseconds From a51600f30ea6bdb53c3ddbe004da7944d771534c Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 13 Jun 2025 11:21:42 +0300 Subject: [PATCH 22/56] UI: Fixed knob and digital gauge decimals setting --- .../gauge/digital-simple-gauge-basic-config.component.ts | 7 +++++-- .../home/components/widget/lib/rpc/knob.component.ts | 4 ++-- .../control/knob-control-widget-settings.component.ts | 2 +- .../gauge/digital-gauge-widget-settings.component.ts | 6 +++++- .../app/shared/models/widget/rpc/knob.component.models.ts | 4 ++-- 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/gauge/digital-simple-gauge-basic-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/gauge/digital-simple-gauge-basic-config.component.ts index 67497b0b6e..9e2c46ec13 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/gauge/digital-simple-gauge-basic-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/gauge/digital-simple-gauge-basic-config.component.ts @@ -30,7 +30,7 @@ import { getTimewindowConfig, setTimewindowConfig } from '@home/components/widget/config/timewindow-config-panel.component'; -import { formatValue, isUndefined } from '@core/utils'; +import { formatValue, isDefinedAndNotNull, isUndefined } from '@core/utils'; import { Component } from '@angular/core'; import { convertLevelColorsSettingsToColorProcessor, @@ -115,7 +115,7 @@ export class DigitalSimpleGaugeBasicConfigComponent extends BasicWidgetConfigCom minMaxColor: [settings.minMaxFont?.color, []], showValue: [settings.showValue, []], - decimals: [configData.config.decimals, []], + decimals: [configData.config.decimals || settings.decimals, []], units: [configData.config.units, []], valueFont: [settings.valueFont, []], valueColor: [settings.valueFont?.color, []], @@ -157,6 +157,9 @@ export class DigitalSimpleGaugeBasicConfigComponent extends BasicWidgetConfigCom this.widgetConfig.config.settings.showValue = config.showValue; this.widgetConfig.config.units = config.units; this.widgetConfig.config.decimals = config.decimals; + if (isDefinedAndNotNull(this.widgetConfig.config.settings.decimals)) { + this.widgetConfig.config.settings.decimals = null; + } this.widgetConfig.config.settings.valueFont = config.valueFont; this.widgetConfig.config.settings.valueFont.color = config.valueColor; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.ts index 8c1fc3ab2a..1b93538b6c 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.ts @@ -145,7 +145,7 @@ export class KnobComponent extends BasicActionWidgetComponent implements OnInit, this.valueFormat = ValueFormatProcessor.fromSettings(this.ctx.$injector, { units: this.ctx.units, - decimals: this.ctx.decimals, + decimals: this.ctx.decimals || 0, showZeroDecimals: true }); @@ -299,7 +299,7 @@ export class KnobComponent extends BasicActionWidgetComponent implements OnInit, } private turn(ratio: number) { - this.newValue = Number((this.minValue + (this.maxValue - this.minValue) * ratio).toFixed(this.ctx.decimals)); + this.newValue = Number((this.minValue + (this.maxValue - this.minValue) * ratio).toFixed(this.ctx.decimals || 0)); if (this.canvasBar.value !== this.newValue) { this.canvasBar.value = this.newValue; } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/control/knob-control-widget-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/control/knob-control-widget-settings.component.ts index e5c57474c2..26a5d6cd4e 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/control/knob-control-widget-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/control/knob-control-widget-settings.component.ts @@ -57,7 +57,7 @@ export class KnobControlWidgetSettingsComponent extends WidgetSettingsComponent protected prepareInputSettings(settings: WidgetSettings): WidgetSettings { const knobSettings = prepareKnobSettings(deepClone(settings) as any) as WidgetSettings; - knobSettings.valueDecimals = this.widgetConfig?.config?.decimals ?? 2; + knobSettings.valueDecimals = this.widgetConfig?.config?.decimals; knobSettings.valueUnits = deepClone(this.widgetConfig?.config?.units); return super.prepareInputSettings(knobSettings); } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/gauge/digital-gauge-widget-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/gauge/digital-gauge-widget-settings.component.ts index 7276c0e27d..9f095cdff0 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/gauge/digital-gauge-widget-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/gauge/digital-gauge-widget-settings.component.ts @@ -37,7 +37,7 @@ import { digitalGaugeLayoutTranslations, DigitalGaugeType } from '@home/components/widget/lib/digital-gauge.models'; -import { formatValue } from '@core/utils'; +import { formatValue, isDefinedAndNotNull } from '@core/utils'; import { ColorSettings, ColorType, @@ -247,6 +247,10 @@ export class DigitalGaugeWidgetSettingsComponent extends WidgetSettingsComponent settings.titleFont.color = this.digitalGaugeWidgetSettingsForm.get('titleColor').value; settings.labelFont.color = this.digitalGaugeWidgetSettingsForm.get('labelColor').value; + if (isDefinedAndNotNull(settings.decimals)) { + settings.decimals = null; + } + return settings; } diff --git a/ui-ngx/src/app/shared/models/widget/rpc/knob.component.models.ts b/ui-ngx/src/app/shared/models/widget/rpc/knob.component.models.ts index 47b73f5a08..3ac15bf70c 100644 --- a/ui-ngx/src/app/shared/models/widget/rpc/knob.component.models.ts +++ b/ui-ngx/src/app/shared/models/widget/rpc/knob.component.models.ts @@ -45,7 +45,7 @@ export const knobWidgetDefaultSettings: KnobSettings = { defaultValue: 50, executeRpc: { method: 'getValue', - requestTimeout: 500, + requestTimeout: 5000, requestPersistent: false, persistentPollingInterval: 5000 }, @@ -70,7 +70,7 @@ export const knobWidgetDefaultSettings: KnobSettings = { action: SetValueAction.EXECUTE_RPC, executeRpc: { method: 'setValue', - requestTimeout: 500, + requestTimeout: 5000, requestPersistent: false, persistentPollingInterval: 5000 }, From f7bc348513aee3247941c93931d676d1bcd9481b Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 13 Jun 2025 11:53:33 +0300 Subject: [PATCH 23/56] UI: delete unused settings --- .../gauge/digital-simple-gauge-basic-config.component.ts | 6 +++--- .../home/components/widget/lib/rpc/knob.component.ts | 6 ++++-- .../gauge/digital-gauge-widget-settings.component.ts | 6 +++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/gauge/digital-simple-gauge-basic-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/gauge/digital-simple-gauge-basic-config.component.ts index 9e2c46ec13..aa93a56a4c 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/gauge/digital-simple-gauge-basic-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/gauge/digital-simple-gauge-basic-config.component.ts @@ -30,7 +30,7 @@ import { getTimewindowConfig, setTimewindowConfig } from '@home/components/widget/config/timewindow-config-panel.component'; -import { formatValue, isDefinedAndNotNull, isUndefined } from '@core/utils'; +import { formatValue, isDefined, isUndefined } from '@core/utils'; import { Component } from '@angular/core'; import { convertLevelColorsSettingsToColorProcessor, @@ -157,8 +157,8 @@ export class DigitalSimpleGaugeBasicConfigComponent extends BasicWidgetConfigCom this.widgetConfig.config.settings.showValue = config.showValue; this.widgetConfig.config.units = config.units; this.widgetConfig.config.decimals = config.decimals; - if (isDefinedAndNotNull(this.widgetConfig.config.settings.decimals)) { - this.widgetConfig.config.settings.decimals = null; + if (isDefined(this.widgetConfig.config.settings.decimals)) { + delete this.widgetConfig.config.settings.decimals; } this.widgetConfig.config.settings.valueFont = config.valueFont; this.widgetConfig.config.settings.valueFont.color = config.valueColor; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.ts index 1b93538b6c..f28b5e8f92 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.ts @@ -57,6 +57,7 @@ export class KnobComponent extends BasicActionWidgetComponent implements OnInit, maxValue: number; newValue = 0; + private decimals: number; private startDeg = -1; private currentDeg = 0; private rotation = 0; @@ -143,9 +144,10 @@ export class KnobComponent extends BasicActionWidgetComponent implements OnInit, actionLabel: this.ctx.translate.instant('widgets.slider.on-value-change')}; this.valueSetter = this.createValueSetter(valueChangeSettings); + this.decimals = isDefined(this.ctx.decimals) ? this.ctx.decimals : 0; this.valueFormat = ValueFormatProcessor.fromSettings(this.ctx.$injector, { units: this.ctx.units, - decimals: this.ctx.decimals || 0, + decimals: this.decimals, showZeroDecimals: true }); @@ -299,7 +301,7 @@ export class KnobComponent extends BasicActionWidgetComponent implements OnInit, } private turn(ratio: number) { - this.newValue = Number((this.minValue + (this.maxValue - this.minValue) * ratio).toFixed(this.ctx.decimals || 0)); + this.newValue = Number((this.minValue + (this.maxValue - this.minValue) * ratio).toFixed(this.decimals)); if (this.canvasBar.value !== this.newValue) { this.canvasBar.value = this.newValue; } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/gauge/digital-gauge-widget-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/gauge/digital-gauge-widget-settings.component.ts index 9f095cdff0..1f3ae36089 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/gauge/digital-gauge-widget-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/gauge/digital-gauge-widget-settings.component.ts @@ -37,7 +37,7 @@ import { digitalGaugeLayoutTranslations, DigitalGaugeType } from '@home/components/widget/lib/digital-gauge.models'; -import { formatValue, isDefinedAndNotNull } from '@core/utils'; +import { formatValue, isDefined } from '@core/utils'; import { ColorSettings, ColorType, @@ -247,8 +247,8 @@ export class DigitalGaugeWidgetSettingsComponent extends WidgetSettingsComponent settings.titleFont.color = this.digitalGaugeWidgetSettingsForm.get('titleColor').value; settings.labelFont.color = this.digitalGaugeWidgetSettingsForm.get('labelColor').value; - if (isDefinedAndNotNull(settings.decimals)) { - settings.decimals = null; + if (isDefined(settings.decimals)) { + delete settings.decimals; } return settings; From 722aa6a77bf7b1fc8361806e639392a814410845 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 13 Jun 2025 11:59:08 +0300 Subject: [PATCH 24/56] UI: Fixed set decimals value --- .../basic/gauge/digital-simple-gauge-basic-config.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/gauge/digital-simple-gauge-basic-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/gauge/digital-simple-gauge-basic-config.component.ts index aa93a56a4c..7d348662ae 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/gauge/digital-simple-gauge-basic-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/gauge/digital-simple-gauge-basic-config.component.ts @@ -115,7 +115,7 @@ export class DigitalSimpleGaugeBasicConfigComponent extends BasicWidgetConfigCom minMaxColor: [settings.minMaxFont?.color, []], showValue: [settings.showValue, []], - decimals: [configData.config.decimals || settings.decimals, []], + decimals: [isDefined(configData.config.decimals) ? configData.config.decimals : settings.decimals, []], units: [configData.config.units, []], valueFont: [settings.valueFont, []], valueColor: [settings.valueFont?.color, []], From 7ba8e1366cbbe80981a70bb9d91f61471038335a Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 13 Jun 2025 12:06:22 +0300 Subject: [PATCH 25/56] fixed server failure when cf configuration is invalid --- .../service/cf/DefaultCalculatedFieldCache.java | 1 - .../configuration/CalculatedFieldConfiguration.java | 2 ++ .../cf/DefaultNativeCalculatedFieldRepository.java | 12 ++++++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index ed35d96cb7..3988cb38db 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -70,7 +70,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @AfterStartUp(order = AfterStartUp.CF_READ_CF_SERVICE) public void init() { - //TODO: move to separate place to avoid circular references with the ActorSystemContext (@Lazy for tsSubService) PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> { calculatedFields.putIfAbsent(cf.getId(), cf); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index c53f1fe5f1..ad3d4373ad 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.cf.configuration; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; @@ -36,6 +37,7 @@ import java.util.Map; @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT") }) +@JsonIgnoreProperties(ignoreUnknown = true) public interface CalculatedFieldConfiguration { @JsonIgnore diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index e59ff3f4e6..a01ad517b8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.page.PageData; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.UUID; import java.util.stream.Collectors; @@ -79,7 +80,6 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); long version = row.get("version") != null ? (long) row.get("version") : 0; String debugSettings = (String) row.get("debug_settings"); - Object externalIdObj = row.get("external_id"); CalculatedField calculatedField = new CalculatedField(); calculatedField.setId(new CalculatedFieldId(id)); @@ -89,12 +89,17 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF calculatedField.setType(type); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); + try { + calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); + } catch (Exception e) { + log.error("Invalid configuration for CalculatedField [{}]. Skipping.", id, e); + return null; + } calculatedField.setVersion(version); calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); return calculatedField; - }).collect(Collectors.toList()); + }).filter(Objects::nonNull).collect(Collectors.toList()); return new PageData<>(data, totalPages, totalElements, hasNext); }); } @@ -118,7 +123,6 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); UUID entityId = (UUID) row.get("entity_id"); UUID calculatedFieldId = (UUID) row.get("calculated_field_id"); - JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); calculatedFieldLink.setId(new CalculatedFieldLinkId(id)); From 43429524364e1e7904157924e2cf8b3fa609f944 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 13 Jun 2025 12:17:11 +0300 Subject: [PATCH 26/56] UI: Change mime type for csv export file --- ui-ngx/src/app/shared/import-export/import-export.models.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/shared/import-export/import-export.models.ts b/ui-ngx/src/app/shared/import-export/import-export.models.ts index 2f20a421c3..60df84c737 100644 --- a/ui-ngx/src/app/shared/import-export/import-export.models.ts +++ b/ui-ngx/src/app/shared/import-export/import-export.models.ts @@ -160,7 +160,7 @@ export const ZIP_TYPE: FileType = { }; export const CSV_TYPE: FileType = { - mimeType: 'attachament/csv', + mimeType: 'text/csv', extension: 'csv' }; From edb3ab4ca020b6b03e93a9ab1d5fe0d56d34f3c3 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 13 Jun 2025 12:25:16 +0300 Subject: [PATCH 27/56] removed links to the doc --- .../src/assets/help/en_US/rulenode/switch_node_script_fn.md | 6 ------ .../help/en_US/rulenode/tbel/switch_node_script_fn.md | 6 ------ 2 files changed, 12 deletions(-) diff --git a/ui-ngx/src/assets/help/en_US/rulenode/switch_node_script_fn.md b/ui-ngx/src/assets/help/en_US/rulenode/switch_node_script_fn.md index c4092455fa..49f0dd0a2b 100644 --- a/ui-ngx/src/assets/help/en_US/rulenode/switch_node_script_fn.md +++ b/ui-ngx/src/assets/help/en_US/rulenode/switch_node_script_fn.md @@ -91,11 +91,5 @@ return []; {:copy-code} ``` -
- -You can see real life example, how to use this node in this tutorial: - -- [Data function based on telemetry from 2 devices{:target="_blank"}](${siteBaseUrl}/docs${docPlatformPrefix}/user-guide/rule-engine-2-0/tutorials/function-based-on-telemetry-from-two-devices#delta-temperature-rule-chain) -

diff --git a/ui-ngx/src/assets/help/en_US/rulenode/tbel/switch_node_script_fn.md b/ui-ngx/src/assets/help/en_US/rulenode/tbel/switch_node_script_fn.md index 2461f4cace..77da936c73 100644 --- a/ui-ngx/src/assets/help/en_US/rulenode/tbel/switch_node_script_fn.md +++ b/ui-ngx/src/assets/help/en_US/rulenode/tbel/switch_node_script_fn.md @@ -91,11 +91,5 @@ return []; {:copy-code} ``` -
- -You can see real life example, how to use this node in this tutorial: - -- [Data function based on telemetry from 2 devices{:target="_blank"}](${siteBaseUrl}/docs${docPlatformPrefix}/user-guide/rule-engine-2-0/tutorials/function-based-on-telemetry-from-two-devices#delta-temperature-rule-chain) -

From 8ae39d4e9416d204453c91d66e04e2713711c1b3 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 13 Jun 2025 12:59:25 +0300 Subject: [PATCH 28/56] moved check to higher level --- .../server/service/cf/DefaultCalculatedFieldCache.java | 6 ++++-- .../dao/sql/cf/DefaultNativeCalculatedFieldRepository.java | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 3988cb38db..78042ffc0e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -72,8 +72,10 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { public void init() { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> { - calculatedFields.putIfAbsent(cf.getId(), cf); - actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf)); + if (cf != null) { + calculatedFields.putIfAbsent(cf.getId(), cf); + actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf)); + } }); calculatedFields.values().forEach(cf -> { entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index a01ad517b8..bbce4e2721 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -38,7 +38,6 @@ import org.thingsboard.server.common.data.page.PageData; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.UUID; import java.util.stream.Collectors; @@ -99,7 +98,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); return calculatedField; - }).filter(Objects::nonNull).collect(Collectors.toList()); + }).collect(Collectors.toList()); return new PageData<>(data, totalPages, totalElements, hasNext); }); } From 513566bd7b666f9712c555d1ddf5c6f46c92f2c9 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 23 May 2025 11:13:18 +0300 Subject: [PATCH 29/56] Docker file improvements to support installation without mounting the configuration folder (cherry picked from commit a6997cb4a1e61ebae64ab05a63d8dff4b6f0dded) --- msa/tb-node/docker/Dockerfile | 4 ++- msa/tb-node/docker/logback.xml | 38 +++++++++++++++++++++++++++++ msa/tb-node/docker/start-tb-node.sh | 18 ++++++++------ 3 files changed, 52 insertions(+), 8 deletions(-) create mode 100644 msa/tb-node/docker/logback.xml diff --git a/msa/tb-node/docker/Dockerfile b/msa/tb-node/docker/Dockerfile index dfca56acc7..013a37ef9c 100644 --- a/msa/tb-node/docker/Dockerfile +++ b/msa/tb-node/docker/Dockerfile @@ -16,12 +16,14 @@ FROM thingsboard/openjdk17:bookworm-slim -COPY start-tb-node.sh ${pkg.name}.deb /tmp/ +COPY logback.xml start-tb-node.sh ${pkg.name}.deb /tmp/ RUN chmod a+x /tmp/*.sh \ && mv /tmp/start-tb-node.sh /usr/bin && \ (yes | dpkg -i /tmp/${pkg.name}.deb) && \ rm /tmp/${pkg.name}.deb && \ + mv /tmp/logback.xml ${pkg.installFolder}/conf && \ + chown -R ${pkg.user}:${pkg.user} ${pkg.installFolder}/conf/logback.xml && \ (systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :) && \ chown -R ${pkg.user}:${pkg.user} /tmp && \ chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar diff --git a/msa/tb-node/docker/logback.xml b/msa/tb-node/docker/logback.xml new file mode 100644 index 0000000000..269cb89396 --- /dev/null +++ b/msa/tb-node/docker/logback.xml @@ -0,0 +1,38 @@ + + + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + + diff --git a/msa/tb-node/docker/start-tb-node.sh b/msa/tb-node/docker/start-tb-node.sh index 7de30564c9..77221398be 100755 --- a/msa/tb-node/docker/start-tb-node.sh +++ b/msa/tb-node/docker/start-tb-node.sh @@ -15,14 +15,21 @@ # limitations under the License. # -CONF_FOLDER="/config" jarfile=${pkg.installFolder}/bin/${pkg.name}.jar configfile=${pkg.name}.conf run_user=${pkg.user} -source "${CONF_FOLDER}/${configfile}" +CONF_FOLDER="/config" +if [ -d "${CONF_FOLDER}" ]; then + LOGGING_CONFIG="${CONF_FOLDER}/logback.xml" + source "${CONF_FOLDER}/${configfile}" + export LOADER_PATH=${CONF_FOLDER},${LOADER_PATH} +else + CONF_FOLDER="/usr/share/${pkg.name}/conf" + LOGGING_CONFIG="/usr/share/${pkg.name}/conf/logback.xml" + source "${CONF_FOLDER}/${configfile}" +fi -export LOADER_PATH=/config,${LOADER_PATH} cd ${pkg.installFolder}/bin @@ -38,7 +45,6 @@ if [ "$INSTALL_TB" == "true" ]; then exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.ThingsboardInstallApplication \ -Dinstall.load_demo=${loadDemo} \ - -Dspring.jpa.hibernate.ddl-auto=none \ -Dinstall.upgrade=false \ -Dlogging.config=/usr/share/thingsboard/bin/install/logback.xml \ org.springframework.boot.loader.launch.PropertiesLauncher @@ -51,7 +57,6 @@ elif [ "$UPGRADE_TB" == "true" ]; then fromVersion="${FROM_VERSION// }" exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.ThingsboardInstallApplication \ - -Dspring.jpa.hibernate.ddl-auto=none \ -Dinstall.upgrade=true \ -Dinstall.upgrade.from_version=${fromVersion} \ -Dlogging.config=/usr/share/thingsboard/bin/install/logback.xml \ @@ -62,8 +67,7 @@ else echo "Starting '${project.name}' ..." exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.ThingsboardServerApplication \ - -Dspring.jpa.hibernate.ddl-auto=none \ - -Dlogging.config=/config/logback.xml \ + -Dlogging.config=${LOGGING_CONFIG} \ org.springframework.boot.loader.launch.PropertiesLauncher fi From 402cf9b5aafd75f11017afcfee263b037feaeeba Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 13 Jun 2025 14:13:01 +0200 Subject: [PATCH 30/56] CVE-2025-49146 postgresql.driver 4.7.5 -> 4.7.7 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0260dc09be..5093c19435 100755 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ 1.19.0 1.78.1 2.0.1 - 42.7.5 + 42.7.7 org/thingsboard/server/gen/**/*, org/thingsboard/server/extensions/core/plugin/telemetry/gen/**/* From 6e4ee1eb44a049cf401b51c66bf469bf61f721d0 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 13 Jun 2025 14:46:10 +0200 Subject: [PATCH 31/56] CVE-2025-27817 kafka client 3.7.2 -> 3.9.1 (NetworkReceive.java has no code changes in the Kafka upstream) --- .../java/org/apache/kafka/common/network/NetworkReceive.java | 4 ++-- pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/apache/kafka/common/network/NetworkReceive.java b/application/src/main/java/org/apache/kafka/common/network/NetworkReceive.java index 80192520ca..88a03f5fc8 100644 --- a/application/src/main/java/org/apache/kafka/common/network/NetworkReceive.java +++ b/application/src/main/java/org/apache/kafka/common/network/NetworkReceive.java @@ -103,13 +103,13 @@ public class NetworkReceive implements Receive { if (maxSize != UNLIMITED && receiveSize > maxSize) { throw new ThingsboardKafkaClientError("Invalid receive (size = " + receiveSize + " larger than " + maxSize + ")"); } - requestedBufferSize = receiveSize; //may be 0 for some payloads (SASL) + requestedBufferSize = receiveSize; // may be 0 for some payloads (SASL) if (receiveSize == 0) { buffer = EMPTY_BUFFER; } } } - if (buffer == null && requestedBufferSize != -1) { //we know the size we want but havent been able to allocate it yet + if (buffer == null && requestedBufferSize != -1) { // we know the size we want but haven't been able to allocate it yet if (requestedBufferSize > TB_LOG_REQUESTED_BUFFER_SIZE) { String stackTrace = Arrays.stream(Thread.currentThread().getStackTrace()).map(StackTraceElement::toString).collect(Collectors.joining("|")); log.error("Allocating buffer of size {} for source {}", requestedBufferSize, source); diff --git a/pom.xml b/pom.xml index 5093c19435..66b153d63a 100755 --- a/pom.xml +++ b/pom.xml @@ -113,7 +113,7 @@ - 3.7.2 + 3.9.1 8.10.1 3.5.3 2.2 From 6a5fd0d45c171a06048c39bb5e2f58a038304627 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 13 Jun 2025 15:32:44 +0200 Subject: [PATCH 32/56] CVE-2024-12798 logback 1.5.5 -> 1.5.18 --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 66b153d63a..88efe6723f 100755 --- a/pom.xml +++ b/pom.xml @@ -52,9 +52,9 @@ 6.3.8 5.1.5 0.12.5 - 2.0.13 - 2.23.1 - 1.5.5 + 2.0.17 + 2.24.3 + 1.5.18 0.10 4.17.0 4.2.25 From 1d1a72d54dd0678fca776f555fa0ed6fda0a8aae Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 13 Jun 2025 15:35:06 +0200 Subject: [PATCH 33/56] CVE-2024-51504 org.apache.zookeeper:zookeeper@3.9.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 88efe6723f..011b1c967b 100755 --- a/pom.xml +++ b/pom.xml @@ -83,7 +83,7 @@ 2.3.32 2.0.1 5.6.0 - 3.9.2 + 3.9.3 3.25.5 1.63.0 1.2.6 From b69460ab01f86edbe81c5aef5f8220dcc1678942 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 13 Jun 2025 15:42:13 +0200 Subject: [PATCH 34/56] CVE-2025-4949 org.eclipse.jgit:org.eclipse.jgit@6.10.1.202505221210-r --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 011b1c967b..fd21733bd3 100755 --- a/pom.xml +++ b/pom.xml @@ -158,7 +158,7 @@ 2.12.0 3.0.2 - 6.9.0.202403050737-r + 6.10.1.202505221210-r 0.4.8 1.0.0 From 14b3df350260142f856f0e62a1974e92dff981e7 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 13 Jun 2025 15:44:51 +0200 Subject: [PATCH 35/56] CVE-2025-46701 org.apache.tomcat.embed:tomcat-embed-core@10.1.42 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fd21733bd3..94cdff0e9f 100755 --- a/pom.xml +++ b/pom.xml @@ -42,7 +42,7 @@ 4.0.2 2.4.0-b180830.0359 4.0.5 - 10.1.40 + 10.1.42 2.5.2 3.2.12 3.2.12 From 56da48e2075743f212ec4613f07e90fa07eee6ef Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 13 Jun 2025 15:49:47 +0200 Subject: [PATCH 36/56] CVE-2025-41234 org.springframework:spring-web@6.1.15 -> org.springframework:spring-web@6.1.21 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 94cdff0e9f..f38ed581a1 100755 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ 3.2.12 3.2.12 3.2.12 - 6.1.15 + 6.1.21 6.2.11 6.3.8 5.1.5 @@ -1173,7 +1173,7 @@ org.springframework.boot - spring-boot-starter + spring-boot-starter ${spring-boot.version} From a39547ddeedef581aad8cd57870c41b5dd224ee3 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 13 Jun 2025 16:08:11 +0200 Subject: [PATCH 37/56] CVE-2025-22234 org.springframework.security:spring-security-crypto@6.3.9 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f38ed581a1..3e0a67ab0d 100755 --- a/pom.xml +++ b/pom.xml @@ -49,7 +49,7 @@ 3.2.12 6.1.21 6.2.11 - 6.3.8 + 6.3.9 5.1.5 0.12.5 2.0.17 From 29211774f590b1ce1f8ebbc56a30f67c14dfb22f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 13 Jun 2025 17:19:54 +0300 Subject: [PATCH 38/56] Minor refactoring --- .../DefaultCalculatedFieldQueueService.java | 7 +++-- .../cf/ctx/state/CalculatedFieldCtx.java | 29 +++++++------------ 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index a423eb6669..fc5d75be56 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -41,6 +41,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; @@ -200,11 +201,11 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS List versions = result != null ? result.getVersions() : Collections.emptyList(); for (int i = 0; i < entries.size(); i++) { - TsKvEntry tsKvEntry = entries.get(i); + TsKvProto.Builder tsProtoBuilder = toTsKvProto(entries.get(i)).toBuilder(); if (versions != null && !versions.isEmpty() && versions.get(i) != null) { - tsKvEntry.setVersion(versions.get(i)); + tsProtoBuilder.setVersion(versions.get(i)); } - telemetryMsg.addTsData(toTsKvProto(tsKvEntry)); + telemetryMsg.addTsData(tsProtoBuilder.build()); } return ToCalculatedFieldMsg.newBuilder().setTelemetryMsg(telemetryMsg).build(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 0008874779..2e3321eece 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -185,10 +185,8 @@ public class CalculatedFieldCtx { return false; } - var lookupKey = new ReferencedEntityKey(null, ArgumentType.ATTRIBUTE, scope); for (AttributeKvEntry attrKv : values) { - lookupKey.setKey(attrKv.getKey()); - if (argMap.containsKey(lookupKey)) { + if (argMap.containsKey(new ReferencedEntityKey(attrKv.getKey(), ArgumentType.ATTRIBUTE, scope))) { return true; } } @@ -201,17 +199,15 @@ public class CalculatedFieldCtx { return false; } - var lookupKey = new ReferencedEntityKey(null, null, null); for (TsKvEntry tsKv : values) { - lookupKey.setKey(tsKv.getKey()); - lookupKey.setType(ArgumentType.TS_LATEST); - if (argMap.containsKey(lookupKey)) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_LATEST, null); + if (argMap.containsKey(latestKey)) { return true; } - lookupKey.setType(ArgumentType.TS_ROLLING); - if (argMap.containsKey(lookupKey)) { + ReferencedEntityKey rollingKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_ROLLING, null); + if (argMap.containsKey(rollingKey)) { return true; } } @@ -232,10 +228,9 @@ public class CalculatedFieldCtx { return false; } - var lookupKey = new ReferencedEntityKey(null, ArgumentType.ATTRIBUTE, scope); for (String key : keys) { - lookupKey.setKey(key); - if (argMap.containsKey(lookupKey)) { + ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); + if (argMap.containsKey(attrKey)) { return true; } } @@ -248,17 +243,15 @@ public class CalculatedFieldCtx { return false; } - var lookupKey = new ReferencedEntityKey(null, null, null); for (String key : keys) { - lookupKey.setKey(key); - lookupKey.setType(ArgumentType.TS_LATEST); - if (argMap.containsKey(lookupKey)) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); + if (argMap.containsKey(latestKey)) { return true; } - lookupKey.setType(ArgumentType.TS_ROLLING); - if (argMap.containsKey(lookupKey)) { + ReferencedEntityKey rollingKey = new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null); + if (argMap.containsKey(rollingKey)) { return true; } } From bb570add28d7e93d0834d67b4cdf7d553a9ab9e7 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 13 Jun 2025 18:04:42 +0300 Subject: [PATCH 39/56] test fixes --- .../cf/AbstractCalculatedFieldStateService.java | 2 ++ .../ctx/state/KafkaCalculatedFieldStateService.java | 13 ++++++++++++- .../cf/ctx/state/SimpleCalculatedFieldState.java | 3 +-- .../DefaultTelemetrySubscriptionService.java | 6 +++--- application/src/main/resources/thingsboard.yml | 2 +- 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java index f1cb25c6fa..70b41f069e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.cf; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -39,6 +40,7 @@ import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; public abstract class AbstractCalculatedFieldStateService implements CalculatedFieldStateService { @Autowired + @Lazy private ActorSystemContext actorSystemContext; protected QueueStateService, TbProtoQueueMsg> stateService; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index 6641a06b1d..2b52892744 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -30,7 +30,9 @@ import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgHeaders; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; import org.thingsboard.server.queue.common.state.KafkaQueueStateService; @@ -109,7 +111,16 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta if (stateMsgProto == null) { putStateId(msg.getHeaders(), stateId); } - stateProducer.send(tpi, stateId.toKey(), msg, null); + stateProducer.send(tpi, stateId.toKey(), msg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + } + + @Override + public void onFailure(Throwable t) { + log.error("Failed to send state message: {}", stateId, t); + } + }); callback.onSuccess(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 577ff80219..026461bd48 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -61,8 +61,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { double value = switch (kvEntry.getDataType()) { case LONG -> kvEntry.getLongValue().map(Long::doubleValue).orElseThrow(); case DOUBLE -> kvEntry.getDoubleValue().orElseThrow(); - case BOOLEAN -> kvEntry.getBooleanValue().map(b -> b ? 1.0 : 0.0).orElseThrow(); - case STRING, JSON -> Double.parseDouble(kvEntry.getValueAsString()); + case BOOLEAN, STRING, JSON -> Double.parseDouble(kvEntry.getValueAsString()); }; expr.setVariable(entry.getKey(), value); } catch (NumberFormatException e) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index a14e052659..69b41addf9 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -95,8 +95,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Value("${sql.ts.value_no_xss_validation:false}") private boolean valueNoXssValidation; - @Value("${sql.ts.thread_pool_size:12}") - private int threadPoolSize; + @Value("${sql.ts.callback_thread_pool_size:12}") + private int callbackThreadPoolSize; public DefaultTelemetrySubscriptionService(AttributesService attrService, TimeseriesService tsService, @@ -117,7 +117,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @PostConstruct public void initExecutor() { super.initExecutor(); - tsCallBackExecutor = ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "ts-service-ts-callback"); + tsCallBackExecutor = ThingsBoardExecutors.newWorkStealingPool(callbackThreadPoolSize, "ts-service-ts-callback"); } @Override diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 958ad2f84c..c3d28f9fce 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -346,7 +346,7 @@ sql: stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}" # Interval in milliseconds for printing timeseries insert statistic batch_threads: "${SQL_TS_BATCH_THREADS:3}" # batch thread count has to be a prime number like 3 or 5 to gain perfect hash distribution value_no_xss_validation: "${SQL_TS_VALUE_NO_XSS_VALIDATION:false}" # If true telemetry values will be checked for XSS vulnerability - thread_pool_size: "${SQL_TS_THREAD_POOL_SIZE:12}" # Thread pool size for telemetry callback executor + callback_thread_pool_size: "${SQL_TS_CALLBACK_THREAD_POOL_SIZE:12}" # Thread pool size for telemetry callback executor ts_latest: batch_size: "${SQL_TS_LATEST_BATCH_SIZE:1000}" # Batch size for persisting latest telemetry updates batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:50}" # Maximum timeout for latest telemetry entries queue polling. The value set in milliseconds From be789ebd1ffa6f968247df3bb501e1d2bd87dcbc Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 13 Jun 2025 18:37:25 +0200 Subject: [PATCH 40/56] downgrade logback as the least compatible version with Spring Boot 3.2.12 is logback 1.5.6 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3e0a67ab0d..4ddbe5ff0b 100755 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,7 @@ 0.12.5 2.0.17 2.24.3 - 1.5.18 + 1.5.6 0.10 4.17.0 4.2.25 From 916dc6cb6e2caffd82a6fd25c1471ddf6cee2ed3 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 16 Jun 2025 07:39:23 +0300 Subject: [PATCH 41/56] added test --- .../ctx/state/SimpleCalculatedFieldState.java | 3 ++- .../state/SimpleCalculatedFieldStateTest.java | 20 ++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 026461bd48..577ff80219 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -61,7 +61,8 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { double value = switch (kvEntry.getDataType()) { case LONG -> kvEntry.getLongValue().map(Long::doubleValue).orElseThrow(); case DOUBLE -> kvEntry.getDoubleValue().orElseThrow(); - case BOOLEAN, STRING, JSON -> Double.parseDouble(kvEntry.getValueAsString()); + case BOOLEAN -> kvEntry.getBooleanValue().map(b -> b ? 1.0 : 0.0).orElseThrow(); + case STRING, JSON -> Double.parseDouble(kvEntry.getValueAsString()); }; expr.setVariable(entry.getKey(), value); } catch (NumberFormatException e) { diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index f0059e4f6f..c1616a85db 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; @@ -143,7 +144,7 @@ public class SimpleCalculatedFieldStateTest { } @Test - void testPerformCalculationWhenPassedNotNumber() { + void testPerformCalculationWhenPassedString() { state.arguments = new HashMap<>(Map.of( "key1", key1ArgEntry, "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 9, new StringDataEntry("key2", "string"), 124L), @@ -155,6 +156,23 @@ public class SimpleCalculatedFieldStateTest { .hasMessage("Argument 'key2' is not a number."); } + @Test + void testPerformCalculationWhenPassedBoolean() throws ExecutionException, InterruptedException { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 9, new BooleanDataEntry("key2", true), 124L),// true is parsed as 1 + "key3", key3ArgEntry + )); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + Output output = getCalculatedFieldConfig().getOutput(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 35))); + } + @Test void testPerformCalculationWhenDecimalsByDefault() throws ExecutionException, InterruptedException { state.arguments = new HashMap<>(Map.of( From aea21e8dad84a23318128941f4c974f42c4298e3 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Mon, 16 Jun 2025 16:15:20 +0300 Subject: [PATCH 42/56] UI: make credentials expansion paned expanded by default --- .../rule-node/common/credentials-config.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/credentials-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/common/credentials-config.component.html index e0d1714e2c..7f7d50db3e 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/common/credentials-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/credentials-config.component.html @@ -16,7 +16,7 @@ -->
- + rule-node-config.credentials From dcb8fdd053e7d0d02710d7427819118df85d116e Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 10 Jun 2025 19:19:38 +0300 Subject: [PATCH 43/56] EdgeGrpcService - destroy if previous session exists --- .../server/service/edge/rpc/EdgeGrpcService.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java index 73da3694f9..d1448d2345 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java @@ -328,6 +328,9 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i Edge edge = edgeGrpcSession.getEdge(); TenantId tenantId = edge.getTenantId(); log.info("[{}][{}] edge [{}] connected successfully.", tenantId, edgeGrpcSession.getSessionId(), edgeId); + if (sessions.containsKey(edgeId)) { + destroySession(sessions.get(edgeId)); + } sessions.put(edgeId, edgeGrpcSession); final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock()); newEventLock.lock(); @@ -493,7 +496,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } finally { newEventLock.unlock(); } - toRemove.destroy(); + destroySession(toRemove); TenantId tenantId = toRemove.getEdge().getTenantId(); save(tenantId, edgeId, ACTIVITY_STATE, false); long lastDisconnectTs = System.currentTimeMillis(); @@ -506,6 +509,12 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i edgeIdServiceIdCache.evict(edgeId); } + private void destroySession(EdgeGrpcSession session) { + try (session) { + session.destroy(); + } + } + private void save(TenantId tenantId, EdgeId edgeId, String key, long value) { log.debug("[{}][{}] Updating long edge telemetry [{}] [{}]", tenantId, edgeId, key, value); if (persistToTelemetry) { From 2bd7b4d01d51930009633d957de5ce919800d2ad Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 10 Jun 2025 19:22:00 +0300 Subject: [PATCH 44/56] EDQS - fixed relations query in case multiple previous path are present --- .../query/processor/AbstractRelationQueryProcessor.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java index 2842d57ff0..193f3bb22e 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.edqs.query.processor; +import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; import org.thingsboard.server.common.data.permission.QueryContext; import org.thingsboard.server.common.data.query.EntityFilter; @@ -106,7 +107,7 @@ public abstract class AbstractRelationQueryProcessor ext private Set> getEntitiesSet(RelationsRepo relations) { Set> result = new HashSet<>(); - Set processed = new HashSet<>(); + Set processed = new HashSet<>(); Queue tasks = new LinkedList<>(); int maxLvl = getMaxLevel() == 0 ? MAXIMUM_QUERY_LEVEL : Math.max(1, getMaxLevel()); for (UUID uuid : getRootEntities()) { @@ -114,7 +115,7 @@ public abstract class AbstractRelationQueryProcessor ext } while (!tasks.isEmpty()) { RelationSearchTask task = tasks.poll(); - if (processed.add(task.entityId)) { + if (processed.add(task)) { var entityLvl = task.lvl + 1; Set entities = EntitySearchDirection.FROM.equals(getDirection()) ? relations.getFrom(task.entityId) : relations.getTo(task.entityId); if (isFetchLastLevelOnly() && entities.isEmpty() && task.previous != null && check(task.previous)) { @@ -157,6 +158,7 @@ public abstract class AbstractRelationQueryProcessor ext protected abstract boolean check(RelationInfo relationInfo); @RequiredArgsConstructor + @EqualsAndHashCode private static class RelationSearchTask { private final UUID entityId; private final int lvl; From dd9d954c2f232b79cd28d268e141ebab9b71881a Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 10 Jun 2025 19:29:16 +0300 Subject: [PATCH 45/56] EDQS - added SQL like style for filter contains/starts with/ends with --- ...stractEntityProfileNameQueryProcessor.java | 2 +- .../AbstractEntityProfileQueryProcessor.java | 2 +- .../processor/EntityNameQueryProcessor.java | 2 +- .../server/edqs/util/RepositoryUtils.java | 60 +++++++++++++------ .../server/edqs/repo/RepositoryUtilsTest.java | 40 ++++++++++++- 5 files changed, 84 insertions(+), 22 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java index b78e49879e..f881d616ef 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java @@ -36,7 +36,7 @@ public abstract class AbstractEntityProfileNameQueryProcessor(getProfileNames(this.filter)); - pattern = RepositoryUtils.toSqlLikePattern(getEntityNameFilter(filter)); + pattern = RepositoryUtils.toContainsSqlLikePattern(getEntityNameFilter(filter)); } protected abstract String getEntityNameFilter(T filter); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java index 301ead7c63..9d043ff6fc 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java @@ -43,7 +43,7 @@ public abstract class AbstractEntityProfileQueryProcessor value.equals(predicateValue); - case STARTS_WITH -> value.startsWith(predicateValue); - case ENDS_WITH -> value.endsWith(predicateValue); + case STARTS_WITH -> toStartsWithSqlLikePattern(predicateValue).matcher(value).matches(); + case ENDS_WITH -> toEndsWithSqlLikePattern(predicateValue).matcher(value).matches(); case NOT_EQUAL -> !value.equals(predicateValue); - case CONTAINS -> value.contains(predicateValue); - case NOT_CONTAINS -> !value.contains(predicateValue); + case CONTAINS -> toContainsSqlLikePattern(predicateValue).matcher(value).matches(); + case NOT_CONTAINS -> !toContainsSqlLikePattern(predicateValue).matcher(value).matches(); case IN -> equalsAny(value, splitByCommaWithoutQuotes(predicateValue)); case NOT_IN -> !equalsAny(value, splitByCommaWithoutQuotes(predicateValue)); }; @@ -304,6 +304,15 @@ public class RepositoryUtils { return true; } else if (filterPredicates.getOperation() == OR) { for (KeyFilterPredicate filterPredicate : filterPredicates.getPredicates()) { + + // Emulate the SQL-like behavior of ThingsBoard's Entity Data Query service: + // for COMPLEX filters, return no results if filter value is empty + if (filterPredicate instanceof StringFilterPredicate stringFilterPredicate) { + if (StringUtils.isEmpty(stringFilterPredicate.getValue().getValue())) { + continue; + } + } + if (simpleKeyFilter.check(value, filterPredicate)) { return true; } @@ -314,23 +323,38 @@ public class RepositoryUtils { } } - public static Pattern toSqlLikePattern(String nameFilter) { - if (StringUtils.isNotBlank(nameFilter)) { - boolean percentSymbolOnStart = nameFilter.startsWith("%"); - boolean percentSymbolOnEnd = nameFilter.endsWith("%"); - if (percentSymbolOnStart) { - nameFilter = nameFilter.substring(1); - } - if (percentSymbolOnEnd) { - nameFilter = nameFilter.substring(0, nameFilter.length() - 1); - } - if (percentSymbolOnStart || percentSymbolOnEnd) { - return Pattern.compile((percentSymbolOnStart ? ".*" : "") + Pattern.quote(nameFilter) + (percentSymbolOnEnd ? ".*" : ""), Pattern.CASE_INSENSITIVE); + public static Pattern toContainsSqlLikePattern(String filter) { + if (StringUtils.isNotBlank(filter)) { + return toSqlLikePattern(filter, ".*", ".*"); + } + return null; + } + + private static Pattern toStartsWithSqlLikePattern(String filter) { + return toSqlLikePattern(filter, "^", ".*"); + } + + private static Pattern toEndsWithSqlLikePattern(String filter) { + return toSqlLikePattern(filter, ".*", "$"); + } + + private static Pattern toSqlLikePattern(String value, String prefix, String suffix ) { + if (value.contains("%") || value.contains("_")) { + String regexValue = value + .replace("_", ".") + .replace("%", ".*"); + String regex; + if ("^".equals(prefix)) { + regex = "^" + regexValue + (regexValue.endsWith(".*") ? "" : ".*"); + } else if ("$".equals(suffix)) { + regex = (regexValue.startsWith(".*") ? "" : ".*") + regexValue + "$"; } else { - return Pattern.compile(Pattern.quote(nameFilter) + ".*", Pattern.CASE_INSENSITIVE); + regex = (regexValue.startsWith(".*") ? "" : ".*") + regexValue + (regexValue.endsWith(".*") ? "" : ".*"); } + return Pattern.compile(regex, Pattern.CASE_INSENSITIVE); + } else { + return Pattern.compile(prefix + Pattern.quote(value) + suffix, Pattern.CASE_INSENSITIVE); } - return null; } @FunctionalInterface diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java index 6c7444c92a..fa3784ca19 100644 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java @@ -70,7 +70,45 @@ public class RepositoryUtilsTest { Arguments.of("loranet 123", getNameFilter(StringOperation.IN, "loranet 123, loranet 124"), true), Arguments.of("loranet 123", getNameFilter(StringOperation.IN, "loranet 125, loranet 126"), false), Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 125, loranet 126"), true), - Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 123, loranet 126"), false) + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 123, loranet 126"), false), + + // Basic CONTAINS + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%loranet"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet%"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%ranet%"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%123"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%loranx%"), false), + + // Basic STARTS_WITH + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "loranet%"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "lora%"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "lorax%"), false), + + // Basic ENDS_WITH + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "%123"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "%23"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "%124"), false), + + // CONTAINS with _ + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet_123"), true), // '_' = ' ' + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet_12_"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loran_t%"), true), + + // STARTS_WITH with _ + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "loranet_"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "lora__t%"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "lor_net%"), true), + + // ENDS_WITH with _ + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "_23"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "_2_"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "_3"), true), + + // Mixed patterns + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "lora__t 1%"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "lora%net%3"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%o_anet%2_3"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "lora___ ___"), true) ); } From e752902cfc2d8baae5ada011042ab91c6f8e08ae Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 11 Jun 2025 12:16:56 +0300 Subject: [PATCH 46/56] Fixed case sensitive matcher --- .../org/thingsboard/server/edqs/util/RepositoryUtils.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java index 1a47c4814f..1550c1df30 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java @@ -54,7 +54,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -338,7 +337,7 @@ public class RepositoryUtils { return toSqlLikePattern(filter, ".*", "$"); } - private static Pattern toSqlLikePattern(String value, String prefix, String suffix ) { + private static Pattern toSqlLikePattern(String value, String prefix, String suffix) { if (value.contains("%") || value.contains("_")) { String regexValue = value .replace("_", ".") @@ -351,9 +350,9 @@ public class RepositoryUtils { } else { regex = (regexValue.startsWith(".*") ? "" : ".*") + regexValue + (regexValue.endsWith(".*") ? "" : ".*"); } - return Pattern.compile(regex, Pattern.CASE_INSENSITIVE); + return Pattern.compile(regex); } else { - return Pattern.compile(prefix + Pattern.quote(value) + suffix, Pattern.CASE_INSENSITIVE); + return Pattern.compile(prefix + Pattern.quote(value) + suffix); } } From 3288ec257f8e898a7c2de2ba25fb03991b964386 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 17 Jun 2025 12:37:24 +0300 Subject: [PATCH 47/56] EDQS SQL like query updates to be in sync with legacy --- ...stractEntityProfileNameQueryProcessor.java | 2 +- .../AbstractEntityProfileQueryProcessor.java | 2 +- .../processor/EntityNameQueryProcessor.java | 2 +- .../server/edqs/util/RepositoryUtils.java | 29 ++++++------------- .../server/edqs/repo/RepositoryUtilsTest.java | 2 +- 5 files changed, 13 insertions(+), 24 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java index f881d616ef..95a1a79392 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java @@ -36,7 +36,7 @@ public abstract class AbstractEntityProfileNameQueryProcessor(getProfileNames(this.filter)); - pattern = RepositoryUtils.toContainsSqlLikePattern(getEntityNameFilter(filter)); + pattern = RepositoryUtils.toEntityNameSqlLikePattern(getEntityNameFilter(filter)); } protected abstract String getEntityNameFilter(T filter); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java index 9d043ff6fc..94a7a95404 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java @@ -43,7 +43,7 @@ public abstract class AbstractEntityProfileQueryProcessor value.equals(predicateValue); - case STARTS_WITH -> toStartsWithSqlLikePattern(predicateValue).matcher(value).matches(); - case ENDS_WITH -> toEndsWithSqlLikePattern(predicateValue).matcher(value).matches(); + case STARTS_WITH -> toSqlLikePattern(predicateValue, "^", ".*").matcher(value).matches(); + case ENDS_WITH -> toSqlLikePattern(predicateValue, ".*", "$").matcher(value).matches(); case NOT_EQUAL -> !value.equals(predicateValue); - case CONTAINS -> toContainsSqlLikePattern(predicateValue).matcher(value).matches(); - case NOT_CONTAINS -> !toContainsSqlLikePattern(predicateValue).matcher(value).matches(); + case CONTAINS -> toSqlLikePattern(predicateValue, ".*", ".*").matcher(value).matches(); + case NOT_CONTAINS -> !toSqlLikePattern(predicateValue, ".*", ".*").matcher(value).matches(); case IN -> equalsAny(value, splitByCommaWithoutQuotes(predicateValue)); case NOT_IN -> !equalsAny(value, splitByCommaWithoutQuotes(predicateValue)); }; @@ -322,35 +322,24 @@ public class RepositoryUtils { } } - public static Pattern toContainsSqlLikePattern(String filter) { + public static Pattern toEntityNameSqlLikePattern(String filter) { if (StringUtils.isNotBlank(filter)) { - return toSqlLikePattern(filter, ".*", ".*"); + return toSqlLikePattern(filter, "", ".*"); } return null; } - private static Pattern toStartsWithSqlLikePattern(String filter) { - return toSqlLikePattern(filter, "^", ".*"); - } - - private static Pattern toEndsWithSqlLikePattern(String filter) { - return toSqlLikePattern(filter, ".*", "$"); - } - private static Pattern toSqlLikePattern(String value, String prefix, String suffix) { if (value.contains("%") || value.contains("_")) { String regexValue = value .replace("_", ".") .replace("%", ".*"); - String regex; if ("^".equals(prefix)) { - regex = "^" + regexValue + (regexValue.endsWith(".*") ? "" : ".*"); + regexValue = "^" + regexValue + (regexValue.endsWith(".*") ? "" : ".*"); } else if ("$".equals(suffix)) { - regex = (regexValue.startsWith(".*") ? "" : ".*") + regexValue + "$"; - } else { - regex = (regexValue.startsWith(".*") ? "" : ".*") + regexValue + (regexValue.endsWith(".*") ? "" : ".*"); + regexValue = (regexValue.startsWith(".*") ? "" : ".*") + regexValue + "$"; } - return Pattern.compile(regex); + return Pattern.compile(regexValue); } else { return Pattern.compile(prefix + Pattern.quote(value) + suffix); } diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java index fa3784ca19..f4c256a8cf 100644 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java @@ -73,7 +73,7 @@ public class RepositoryUtilsTest { Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 123, loranet 126"), false), // Basic CONTAINS - Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%loranet"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%loranet"), false), Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet%"), true), Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%ranet%"), true), Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%123"), true), From 969b0fec00cbce93f59ac5ccd6786ffebbc3bc9e Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 17 Jun 2025 15:49:01 +0300 Subject: [PATCH 48/56] Edqs - sql like queries case insensitive --- .../org/thingsboard/server/edqs/util/RepositoryUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java index 3ecbe0d5fa..6a3fb0fe40 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java @@ -339,9 +339,9 @@ public class RepositoryUtils { } else if ("$".equals(suffix)) { regexValue = (regexValue.startsWith(".*") ? "" : ".*") + regexValue + "$"; } - return Pattern.compile(regexValue); + return Pattern.compile(regexValue, Pattern.CASE_INSENSITIVE); } else { - return Pattern.compile(prefix + Pattern.quote(value) + suffix); + return Pattern.compile(prefix + Pattern.quote(value) + suffix, Pattern.CASE_INSENSITIVE); } } From b0a8b9fc25b8a1045f0968a0dac6a5edf6f098c1 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 17 Jun 2025 17:32:04 +0300 Subject: [PATCH 49/56] EDQS - ignore case for name and not ignore for filters --- .../server/edqs/util/RepositoryUtils.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java index 6a3fb0fe40..fab98027ed 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java @@ -324,14 +324,19 @@ public class RepositoryUtils { public static Pattern toEntityNameSqlLikePattern(String filter) { if (StringUtils.isNotBlank(filter)) { - return toSqlLikePattern(filter, "", ".*"); + return toSqlLikePattern(filter, "", ".*", true); } return null; } private static Pattern toSqlLikePattern(String value, String prefix, String suffix) { + return toSqlLikePattern(value, prefix, suffix, false); + } + + private static Pattern toSqlLikePattern(String value, String prefix, String suffix, boolean ignoreCase) { + String regexValue; if (value.contains("%") || value.contains("_")) { - String regexValue = value + regexValue = value .replace("_", ".") .replace("%", ".*"); if ("^".equals(prefix)) { @@ -339,10 +344,10 @@ public class RepositoryUtils { } else if ("$".equals(suffix)) { regexValue = (regexValue.startsWith(".*") ? "" : ".*") + regexValue + "$"; } - return Pattern.compile(regexValue, Pattern.CASE_INSENSITIVE); } else { - return Pattern.compile(prefix + Pattern.quote(value) + suffix, Pattern.CASE_INSENSITIVE); + regexValue = prefix + Pattern.quote(value) + suffix; } + return ignoreCase ? Pattern.compile(regexValue, Pattern.CASE_INSENSITIVE) : Pattern.compile(regexValue); } @FunctionalInterface From df1e2a0c51f25956f1299f91141df654c32889d8 Mon Sep 17 00:00:00 2001 From: yevhenii Date: Tue, 17 Jun 2025 17:49:08 +0300 Subject: [PATCH 50/56] [CP-17334] Edge rule chain error - Skip saving EdgeEvent if edgeId equals originatorEdgeId --- .../server/service/edge/rpc/processor/BaseEdgeProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index 6fcb02e4bc..4eadcfc5e8 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -219,7 +219,7 @@ public abstract class BaseEdgeProcessor implements EdgeProcessor { case CREDENTIALS_UPDATED: case ASSIGNED_TO_CUSTOMER: case UNASSIGNED_FROM_CUSTOMER: - if (edgeId != null) { + if (edgeId != null && !edgeId.equals(originatorEdgeId)) { return saveEdgeEvent(tenantId, edgeId, type, actionType, entityId, body); } else { return processNotificationToRelatedEdges(tenantId, entityId, type, actionType, originatorEdgeId); From 8b641cb973cfc82c2ff57eaf5d2ff58740b2d6fc Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 18 Jun 2025 07:14:38 +0300 Subject: [PATCH 51/56] Fix testCancelJob_whileRunning --- .../java/org/thingsboard/server/service/job/JobManagerTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/service/job/JobManagerTest.java b/application/src/test/java/org/thingsboard/server/service/job/JobManagerTest.java index 8da1be43f1..af499ade31 100644 --- a/application/src/test/java/org/thingsboard/server/service/job/JobManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/job/JobManagerTest.java @@ -165,7 +165,6 @@ public class JobManagerTest extends AbstractControllerTest { await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { Job job = findJobById(jobId); assertThat(job.getStatus()).isEqualTo(JobStatus.CANCELLED); - assertThat(job.getResult().getSuccessfulCount()).isBetween(1, tasksCount - 1); assertThat(job.getResult().getDiscardedCount()).isBetween(1, tasksCount - 1); assertThat(job.getResult().getTotalCount()).isEqualTo(tasksCount); assertThat(job.getResult().getCompletedCount()).isEqualTo(tasksCount); From 78d3a97dd0a5754b58059a7a6c0b8ffd254fdf03 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 18 Jun 2025 07:18:54 +0300 Subject: [PATCH 52/56] Cleanup actor mailbox messages on destroy --- .../org/thingsboard/server/actors/TbActorMailbox.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java index 4b822e0030..6cd28fa98d 100644 --- a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java @@ -237,8 +237,14 @@ public final class TbActorMailbox implements TbActorCtx { try { ready.set(NOT_READY); actor.destroy(stopReason, cause); - highPriorityMsgs.forEach(msg -> msg.onTbActorStopped(stopReason)); - normalPriorityMsgs.forEach(msg -> msg.onTbActorStopped(stopReason)); + highPriorityMsgs.removeIf(msg -> { + msg.onTbActorStopped(stopReason); + return true; + }); + normalPriorityMsgs.removeIf(msg -> { + msg.onTbActorStopped(stopReason); + return true; + }); } catch (Throwable t) { log.warn("[{}] Failed to destroy actor: ", selfId, t); } From 1ec2450c7801f1bffc3dd1790e441fe7789a0978 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 18 Jun 2025 07:22:11 +0300 Subject: [PATCH 53/56] Get rid of rule node configurationBytes --- .../server/common/data/rule/RuleNode.java | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java index d9eff103a7..24d5c20d45 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java @@ -62,9 +62,7 @@ public class RuleNode extends BaseDataWithAdditionalInfo implements @Schema(description = "Version of rule node configuration. ", example = "0") private int configurationVersion; @Schema(description = "JSON with the rule node configuration. Structure depends on the rule node implementation.", implementation = JsonNode.class) - private transient JsonNode configuration; - @JsonIgnore - private byte[] configurationBytes; + private JsonNode configuration; private RuleNodeId externalId; @@ -92,14 +90,6 @@ public class RuleNode extends BaseDataWithAdditionalInfo implements return name; } - public JsonNode getConfiguration() { - return BaseDataWithAdditionalInfo.getJson(() -> configuration, () -> configurationBytes); - } - - public void setConfiguration(JsonNode data) { - setJson(data, json -> this.configuration = json, bytes -> this.configurationBytes = bytes); - } - @Schema(description = "JSON object with the Rule Node Id. " + "Specify this field to update the Rule Node. " + "Referencing non-existing Rule Node Id will cause error. " + From 2c0001e5e01d23adb27f137a175833b6bad3c8bb Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 18 Jun 2025 07:23:56 +0300 Subject: [PATCH 54/56] Remove redundant "transient" usages --- .../server/common/data/AdminSettings.java | 6 +++--- .../server/common/data/BaseData.java | 8 ++++---- .../server/common/data/Dashboard.java | 6 +++--- .../common/data/settings/UserSettings.java | 20 ++----------------- .../server/common/data/widget/WidgetType.java | 4 +++- .../config/ssl/KeystoreSslCredentials.java | 2 +- .../config/ssl/PemSslCredentials.java | 4 ++-- 7 files changed, 18 insertions(+), 32 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java index ea3297e621..6092e65648 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java @@ -32,8 +32,8 @@ public class AdminSettings extends BaseData implements HasTenan @NoXss @Length(fieldName = "key") private String key; - private transient JsonNode jsonValue; - + private JsonNode jsonValue; + public AdminSettings() { super(); } @@ -41,7 +41,7 @@ public class AdminSettings extends BaseData implements HasTenan public AdminSettings(AdminSettingsId id) { super(id); } - + public AdminSettings(AdminSettings adminSettings) { super(adminSettings); this.tenantId = adminSettings.getTenantId(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java b/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java index 10ea83397b..2ac78b04b6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java @@ -25,9 +25,9 @@ public abstract class BaseData extends IdBased implement private static final long serialVersionUID = 5422817607129962637L; public static final ObjectMapper mapper = new ObjectMapper(); - + protected long createdTime; - + public BaseData() { super(); } @@ -35,7 +35,7 @@ public abstract class BaseData extends IdBased implement public BaseData(I id) { super(id); } - + public BaseData(BaseData data) { super(data.getId()); this.createdTime = data.getCreatedTime(); @@ -53,7 +53,7 @@ public abstract class BaseData extends IdBased implement public int hashCode() { final int prime = 31; int result = super.hashCode(); - result = prime * result + (int) (createdTime ^ (createdTime >>> 32)); + result = prime * result + Long.hashCode(createdTime); return result; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index 1deb322d39..820e983d60 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -38,7 +38,7 @@ public class Dashboard extends DashboardInfo implements ExportableEntity settings, () -> settingsBytes); - } + private JsonNode settings; - public void setSettings(JsonNode settings) { - setJson(settings, json -> this.settings = json, bytes -> this.settingsBytes = bytes); - } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetType.java b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetType.java index bdac430ce1..5385a54f74 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetType.java @@ -21,15 +21,17 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.id.WidgetTypeId; import java.util.Optional; +@EqualsAndHashCode(callSuper = true) @Data public class WidgetType extends BaseWidgetType { @Schema(description = "Complex JSON object that describes the widget type", accessMode = Schema.AccessMode.READ_ONLY) - private transient JsonNode descriptor; + private JsonNode descriptor; public WidgetType() { super(); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/KeystoreSslCredentials.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/KeystoreSslCredentials.java index edb1e4694b..97f438a8e1 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/KeystoreSslCredentials.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/KeystoreSslCredentials.java @@ -26,7 +26,7 @@ import java.security.GeneralSecurityException; import java.security.KeyStore; @Data -@EqualsAndHashCode(callSuper = false) +@EqualsAndHashCode(callSuper = true) public class KeystoreSslCredentials extends AbstractSslCredentials { private String type; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/PemSslCredentials.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/PemSslCredentials.java index 6c7e0cf842..4d269f6964 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/PemSslCredentials.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/PemSslCredentials.java @@ -46,7 +46,7 @@ import java.util.List; import java.util.stream.Collectors; @Data -@EqualsAndHashCode(callSuper = false) +@EqualsAndHashCode(callSuper = true) public class PemSslCredentials extends AbstractSslCredentials { private static final String DEFAULT_KEY_ALIAS = "server"; @@ -72,7 +72,7 @@ public class PemSslCredentials extends AbstractSslCredentials { try (InputStream inStream = ResourceUtils.getInputStream(this, this.certFile)) { try (PEMParser pemParser = new PEMParser(new InputStreamReader(inStream))) { Object object; - while((object = pemParser.readObject()) != null) { + while ((object = pemParser.readObject()) != null) { if (object instanceof X509CertificateHolder) { X509Certificate x509Cert = certConverter.getCertificate((X509CertificateHolder) object); certificates.add(x509Cert); From f9523209bff0c29913dea990621708013e351241 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 18 Jun 2025 16:26:25 +0300 Subject: [PATCH 55/56] Fix tests --- .../main/java/org/thingsboard/server/common/data/Dashboard.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index 820e983d60..b5ec74c2d3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -38,7 +38,7 @@ public class Dashboard extends DashboardInfo implements ExportableEntity Date: Thu, 19 Jun 2025 11:13:22 +0300 Subject: [PATCH 56/56] Fix some flaky tests --- .../java/org/thingsboard/server/service/job/JobManagerTest.java | 2 +- .../queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/job/JobManagerTest.java b/application/src/test/java/org/thingsboard/server/service/job/JobManagerTest.java index af499ade31..0069310b5d 100644 --- a/application/src/test/java/org/thingsboard/server/service/job/JobManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/job/JobManagerTest.java @@ -165,7 +165,7 @@ public class JobManagerTest extends AbstractControllerTest { await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { Job job = findJobById(jobId); assertThat(job.getStatus()).isEqualTo(JobStatus.CANCELLED); - assertThat(job.getResult().getDiscardedCount()).isBetween(1, tasksCount - 1); + assertThat(job.getResult().getDiscardedCount()).isBetween(1, tasksCount); assertThat(job.getResult().getTotalCount()).isEqualTo(tasksCount); assertThat(job.getResult().getCompletedCount()).isEqualTo(tasksCount); }); diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java index dd9c4bebcf..09bd02e5a4 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -467,7 +467,7 @@ public class TbRuleEngineQueueConsumerManagerTest { consumerManager.delete(true); - await().atMost(2, TimeUnit.SECONDS) + await().atMost(5, TimeUnit.SECONDS) .untilAsserted(() -> { verify(ruleEngineMsgProducer).send(any(), any(), any()); });