diff --git a/application/src/main/java/org/apache/kafka/common/network/NetworkReceive.java b/application/src/main/java/org/apache/kafka/common/network/NetworkReceive.java index 80192520ca..88a03f5fc8 100644 --- a/application/src/main/java/org/apache/kafka/common/network/NetworkReceive.java +++ b/application/src/main/java/org/apache/kafka/common/network/NetworkReceive.java @@ -103,13 +103,13 @@ public class NetworkReceive implements Receive { if (maxSize != UNLIMITED && receiveSize > maxSize) { throw new ThingsboardKafkaClientError("Invalid receive (size = " + receiveSize + " larger than " + maxSize + ")"); } - requestedBufferSize = receiveSize; //may be 0 for some payloads (SASL) + requestedBufferSize = receiveSize; // may be 0 for some payloads (SASL) if (receiveSize == 0) { buffer = EMPTY_BUFFER; } } } - if (buffer == null && requestedBufferSize != -1) { //we know the size we want but havent been able to allocate it yet + if (buffer == null && requestedBufferSize != -1) { // we know the size we want but haven't been able to allocate it yet if (requestedBufferSize > TB_LOG_REQUESTED_BUFFER_SIZE) { String stackTrace = Arrays.stream(Thread.currentThread().getStackTrace()).map(StackTraceElement::toString).collect(Collectors.joining("|")); log.error("Allocating buffer of size {} for source {}", requestedBufferSize, source); diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 396ecd3771..0125c7c07d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -81,6 +81,7 @@ import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.dao.job.JobService; import org.thingsboard.server.dao.mobile.MobileAppBundleService; import org.thingsboard.server.dao.mobile.MobileAppService; import org.thingsboard.server.dao.nosql.CassandraBufferedRateReadExecutor; @@ -97,7 +98,6 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.rule.RuleNodeStateService; -import org.thingsboard.server.dao.job.JobService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.tenant.TenantService; @@ -320,7 +320,6 @@ public class ActorSystemContext { @Getter private TbEntityViewService tbEntityViewService; - @Lazy @Autowired @Getter private TelemetrySubscriptionService tsSubService; @@ -453,12 +452,10 @@ public class ActorSystemContext { @Getter private ApiLimitService apiLimitService; - @Lazy @Autowired(required = false) @Getter private RateLimitService rateLimitService; - @Lazy @Autowired(required = false) @Getter private DebugModeRateLimitsConfig debugModeRateLimitsConfig; @@ -539,17 +536,14 @@ public class ActorSystemContext { @Getter private EntityService entityService; - @Lazy @Autowired(required = false) @Getter private CalculatedFieldProcessingService calculatedFieldProcessingService; - @Lazy @Autowired(required = false) @Getter private CalculatedFieldStateService calculatedFieldStateService; - @Lazy @Autowired(required = false) @Getter private CalculatedFieldQueueService calculatedFieldQueueService; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 5ea79e42a8..ebc4e60709 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -19,7 +19,6 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.DebugModeUtil; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; @@ -299,7 +298,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM callback.onSuccess(); } if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { - systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, calculationResult.getResult().toString(), null); } } } else { diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 3257b31ca4..2dcb32cf39 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -244,7 +244,7 @@ public class CalculatedFieldController extends BaseController { ); Object[] args = new Object[ctxAndArgNames.size()]; - args[0] = new TbelCfCtx(arguments, getLastUpdateTimestamp(arguments)); + args[0] = new TbelCfCtx(arguments, getLatestTimestamp(arguments)); for (int i = 1; i < ctxAndArgNames.size(); i++) { var arg = arguments.get(ctxAndArgNames.get(i)); if (arg instanceof TbelCfSingleValueArg svArg) { @@ -267,7 +267,7 @@ public class CalculatedFieldController extends BaseController { return result; } - private long getLastUpdateTimestamp(Map arguments) { + private long getLatestTimestamp(Map arguments) { long lastUpdateTimestamp = -1; for (TbelCfArg entry : arguments.values()) { if (entry instanceof TbelCfSingleValueArg singleValueArg) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java index f1cb25c6fa..70b41f069e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.cf; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -39,6 +40,7 @@ import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; public abstract class AbstractCalculatedFieldStateService implements CalculatedFieldStateService { @Autowired + @Lazy private ActorSystemContext actorSystemContext; protected QueueStateService, TbProtoQueueMsg> stateService; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java index eb86220361..f9ec8087ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java @@ -21,6 +21,7 @@ import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.RuleEngineCalculatedFieldQueueService; import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import java.util.List; @@ -35,7 +36,7 @@ public interface CalculatedFieldQueueService extends RuleEngineCalculatedFieldQu */ void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback); - void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); + void pushRequestToQueue(AttributesSaveRequest request, AttributesSaveResult result, FutureCallback callback); void pushRequestToQueue(AttributesDeleteRequest request, List result, FutureCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index ed35d96cb7..95fa7aebb1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -19,7 +19,9 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; +import org.springframework.util.ConcurrentReferenceHashMap; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.cf.CalculatedField; @@ -49,14 +51,13 @@ import java.util.concurrent.locks.ReentrantLock; @RequiredArgsConstructor public class DefaultCalculatedFieldCache implements CalculatedFieldCache { - private static final Integer UNKNOWN_PARTITION = -1; - - private final Lock calculatedFieldFetchLock = new ReentrantLock(); + private final ConcurrentReferenceHashMap calculatedFieldFetchLocks = new ConcurrentReferenceHashMap<>(); private final CalculatedFieldService calculatedFieldService; private final TbelInvokeService tbelInvokeService; - private final ActorSystemContext actorSystemContext; private final ApiLimitService apiLimitService; + @Lazy + private final ActorSystemContext actorSystemContext; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); @@ -70,11 +71,12 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @AfterStartUp(order = AfterStartUp.CF_READ_CF_SERVICE) public void init() { - //TODO: move to separate place to avoid circular references with the ActorSystemContext (@Lazy for tsSubService) PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> { - calculatedFields.putIfAbsent(cf.getId(), cf); - actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf)); + if (cf != null) { + calculatedFields.putIfAbsent(cf.getId(), cf); + actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf)); + } }); calculatedFields.values().forEach(cf -> { entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf); @@ -98,19 +100,20 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public List getCalculatedFieldsByEntityId(EntityId entityId) { - return entityIdCalculatedFields.getOrDefault(entityId, new CopyOnWriteArrayList<>()); + return entityIdCalculatedFields.getOrDefault(entityId, Collections.emptyList()); } @Override public List getCalculatedFieldLinksByEntityId(EntityId entityId) { - return entityIdCalculatedFieldLinks.getOrDefault(entityId, new CopyOnWriteArrayList<>()); + return entityIdCalculatedFieldLinks.getOrDefault(entityId, Collections.emptyList()); } @Override public CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId) { CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); if (ctx == null) { - calculatedFieldFetchLock.lock(); + Lock lock = getFetchLock(calculatedFieldId); + lock.lock(); try { ctx = calculatedFieldsCtx.get(calculatedFieldId); if (ctx == null) { @@ -122,7 +125,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } } } finally { - calculatedFieldFetchLock.unlock(); + lock.unlock(); } } log.trace("[{}] Found calculated field ctx in cache: {}", calculatedFieldId, ctx); @@ -141,7 +144,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - calculatedFieldFetchLock.lock(); + Lock lock = getFetchLock(calculatedFieldId); + lock.lock(); try { CalculatedField calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); if (calculatedField == null) { @@ -163,7 +167,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { .add(configuration.buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)); }); } finally { - calculatedFieldFetchLock.unlock(); + lock.unlock(); } } @@ -187,4 +191,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); } + private Lock getFetchLock(CalculatedFieldId id) { + return calculatedFieldFetchLocks.computeIfAbsent(id, __ -> new ReentrantLock()); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index e9a6cb09aa..f2a6916751 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -25,7 +25,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.math.NumberUtils; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; @@ -66,7 +65,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNot import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; @@ -171,7 +169,7 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP OutputType type = calculatedFieldResult.getType(); TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; - TbMsg msg = TbMsg.newMsg().type(msgType).originator(entityId).previousCalculatedFieldIds(cfIds).metaData(md).data(JacksonUtil.writeValueAsString(calculatedFieldResult.getResult())).build(); + TbMsg msg = TbMsg.newMsg().type(msgType).originator(entityId).previousCalculatedFieldIds(cfIds).metaData(md).data(calculatedFieldResult.getResult().toString()).build(); clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 8289e4db42..fc5d75be56 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; @@ -86,7 +87,10 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); - checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), + var entries = request.getEntries(); + checkEntityAndPushToQueue(tenantId, entityId, + cf -> cf.matches(entries), + cf -> cf.linkMatches(entityId, entries), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -96,10 +100,14 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } @Override - public void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback) { + public void pushRequestToQueue(AttributesSaveRequest request, AttributesSaveResult result, FutureCallback callback) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); - checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), + var entries = request.getEntries(); + var scope = request.getScope(); + checkEntityAndPushToQueue(tenantId, entityId, + cf -> cf.matches(entries, scope), + cf -> cf.linkMatches(entityId, entries, scope), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -112,7 +120,10 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS public void pushRequestToQueue(AttributesDeleteRequest request, List result, FutureCallback callback) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); - checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matchesKeys(result, request.getScope()), cf -> cf.linkMatchesAttrKeys(entityId, result, request.getScope()), + var scope = request.getScope(); + checkEntityAndPushToQueue(tenantId, entityId, + cf -> cf.matchesKeys(result, scope), + cf -> cf.linkMatchesAttrKeys(entityId, result, scope), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -120,8 +131,9 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS public void pushRequestToQueue(TimeseriesDeleteRequest request, List result, FutureCallback callback) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); - - checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matchesKeys(result), cf -> cf.linkMatchesTsKeys(entityId, result), + checkEntityAndPushToQueue(tenantId, entityId, + cf -> cf.matchesKeys(result), + cf -> cf.linkMatchesTsKeys(entityId, result), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -133,7 +145,8 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); if (send) { - clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); + ToCalculatedFieldMsg calculatedFieldMsg = msg.get(); + clusterService.pushMsgToCalculatedFields(tenantId, entityId, calculatedFieldMsg, wrap(callback)); } else { if (callback != null) { callback.onSuccess(null); @@ -142,20 +155,35 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter) { - boolean send = false; - if (supportedReferencedEntities.contains(entityId.getEntityType())) { - send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId).stream().anyMatch(filter); - if (!send) { - send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(getProfileId(tenantId, entityId)).stream().anyMatch(filter); + if (!supportedReferencedEntities.contains(entityId.getEntityType())) { + return false; + } + List entityCfs = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId); + for (CalculatedFieldCtx ctx : entityCfs) { + if (filter.test(ctx)) { + return true; } - if (!send) { - send = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId).stream() - .map(CalculatedFieldLink::getCalculatedFieldId) - .map(calculatedFieldCache::getCalculatedFieldCtx) - .anyMatch(linkedEntityFilter); + } + + EntityId profileId = getProfileId(tenantId, entityId); + if (profileId != null) { + List profileCfs = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(profileId); + for (CalculatedFieldCtx ctx : profileCfs) { + if (filter.test(ctx)) { + return true; + } + } + } + + List links = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId); + for (CalculatedFieldLink link : links) { + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(link.getCalculatedFieldId()); + if (ctx != null && linkedEntityFilter.test(ctx)) { + return true; } } - return send; + + return false; } private EntityId getProfileId(TenantId tenantId, EntityId entityId) { @@ -167,8 +195,6 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { - ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); - CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); List entries = request.getEntries(); @@ -176,25 +202,27 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS for (int i = 0; i < entries.size(); i++) { TsKvProto.Builder tsProtoBuilder = toTsKvProto(entries.get(i)).toBuilder(); - if (result != null) { + if (versions != null && !versions.isEmpty() && versions.get(i) != null) { tsProtoBuilder.setVersion(versions.get(i)); } telemetryMsg.addTsData(tsProtoBuilder.build()); } - msg.setTelemetryMsg(telemetryMsg.build()); - return msg.build(); + return ToCalculatedFieldMsg.newBuilder().setTelemetryMsg(telemetryMsg).build(); } - private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List versions) { + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, AttributesSaveResult result) { ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); + List entries = request.getEntries(); + List versions = result.versions(); + for (int i = 0; i < entries.size(); i++) { AttributeValueProto.Builder attrProtoBuilder = ProtoUtils.toProto(entries.get(i)).toBuilder(); - if (versions != null) { + if (versions != null && !versions.isEmpty() && versions.get(i) != null) { attrProtoBuilder.setVersion(versions.get(i)); } telemetryMsg.addAttrData(attrProtoBuilder.build()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index e4b03b4cab..e21d56b6d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -35,7 +35,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected Map arguments; protected boolean sizeExceedsLimit; - protected long lastUpdateTimestamp = -1; + protected long latestTimestamp = -1; public BaseCalculatedFieldState(List requiredArguments) { this.requiredArguments = requiredArguments; @@ -110,12 +110,14 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected abstract void validateNewEntry(ArgumentEntry newEntry); private void updateLastUpdateTimestamp(ArgumentEntry entry) { + long newTs = this.latestTimestamp; if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - this.lastUpdateTimestamp = singleValueArgumentEntry.getTs(); + newTs = singleValueArgumentEntry.getTs(); } else if (entry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { Map.Entry lastEntry = tsRollingArgumentEntry.getTsRecords().lastEntry(); - this.lastUpdateTimestamp = (lastEntry != null) ? lastEntry.getKey() : System.currentTimeMillis(); + newTs = (lastEntry != null) ? lastEntry.getKey() : System.currentTimeMillis(); } + this.latestTimestamp = Math.max(this.latestTimestamp, newTs); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index a3fdae319d..2e3321eece 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -62,7 +62,7 @@ public class CalculatedFieldCtx { private final List argNames; private Output output; private String expression; - private boolean preserveMsgTs; + private boolean useLatestTs; private TbelInvokeService tbelInvokeService; private CalculatedFieldScriptEngine calculatedFieldScriptEngine; private ThreadLocal customExpression; @@ -96,7 +96,7 @@ public class CalculatedFieldCtx { this.argNames = new ArrayList<>(arguments.keySet()); this.output = configuration.getOutput(); this.expression = configuration.getExpression(); - this.preserveMsgTs = CalculatedFieldType.SIMPLE.equals(calculatedField.getType()) && ((SimpleCalculatedFieldConfiguration) configuration).isPreserveMsgTs(); + this.useLatestTs = CalculatedFieldType.SIMPLE.equals(calculatedField.getType()) && ((SimpleCalculatedFieldConfiguration) configuration).isUseLatestTs(); this.tbelInvokeService = tbelInvokeService; this.maxDataPointsPerRollingArg = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); @@ -181,26 +181,37 @@ public class CalculatedFieldCtx { } private boolean matchesAttributes(Map argMap, List values, AttributeScope scope) { + if (argMap.isEmpty() || values.isEmpty()) { + return false; + } + for (AttributeKvEntry attrKv : values) { - ReferencedEntityKey attrKey = new ReferencedEntityKey(attrKv.getKey(), ArgumentType.ATTRIBUTE, scope); - if (argMap.containsKey(attrKey)) { + if (argMap.containsKey(new ReferencedEntityKey(attrKv.getKey(), ArgumentType.ATTRIBUTE, scope))) { return true; } } + return false; } private boolean matchesTimeSeries(Map argMap, List values) { + if (argMap.isEmpty() || values.isEmpty()) { + return false; + } + for (TsKvEntry tsKv : values) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_LATEST, null); if (argMap.containsKey(latestKey)) { return true; } + ReferencedEntityKey rollingKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_ROLLING, null); if (argMap.containsKey(rollingKey)) { return true; } } + return false; } @@ -213,26 +224,38 @@ public class CalculatedFieldCtx { } private boolean matchesAttributesKeys(Map argMap, List keys, AttributeScope scope) { + if (argMap.isEmpty() || keys.isEmpty()) { + return false; + } + for (String key : keys) { ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); if (argMap.containsKey(attrKey)) { return true; } } + return false; } private boolean matchesTimeSeriesKeys(Map argMap, List keys) { + if (argMap.isEmpty() || keys.isEmpty()) { + return false; + } + for (String key : keys) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); if (argMap.containsKey(latestKey)) { return true; } + ReferencedEntityKey rollingKey = new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null); if (argMap.containsKey(rollingKey)) { return true; } } + return false; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 6eac3358ba..0de354bbb0 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -42,7 +42,7 @@ public interface CalculatedFieldState { Map getArguments(); - long getLastUpdateTimestamp(); + long getLatestTimestamp(); void setRequiredArguments(List requiredArguments); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index ded7adbc34..2b52892744 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -114,18 +114,14 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta stateProducer.send(tpi, stateId.toKey(), msg, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { - if (callback != null) { - callback.onSuccess(); - } } @Override public void onFailure(Throwable t) { - if (callback != null) { - callback.onFailure(t); - } + log.error("Failed to send state message: {}", stateId, t); } }); + callback.onSuccess(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 65ef40330c..84dce627ae 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -66,7 +66,7 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { args.add(arg); } } - args.set(0, new TbelCfCtx(arguments, getLastUpdateTimestamp())); + args.set(0, new TbelCfCtx(arguments, getLatestTimestamp())); ListenableFuture resultFuture = ctx.getCalculatedFieldScriptEngine().executeJsonAsync(args.toArray()); Output output = ctx.getOutput(); return Futures.transform(resultFuture, diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index d0eba5031c..577ff80219 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -58,7 +58,13 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { for (Map.Entry entry : this.arguments.entrySet()) { try { BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getKvEntryValue(); - expr.setVariable(entry.getKey(), Double.parseDouble(kvEntry.getValueAsString())); + double value = switch (kvEntry.getDataType()) { + case LONG -> kvEntry.getLongValue().map(Long::doubleValue).orElseThrow(); + case DOUBLE -> kvEntry.getDoubleValue().orElseThrow(); + case BOOLEAN -> kvEntry.getBooleanValue().map(b -> b ? 1.0 : 0.0).orElseThrow(); + case STRING, JSON -> Double.parseDouble(kvEntry.getValueAsString()); + }; + expr.setVariable(entry.getKey(), value); } catch (NumberFormatException e) { throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); } @@ -68,7 +74,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { Output output = ctx.getOutput(); Object result = formatResult(expressionResult, output.getDecimalsByDefault()); - JsonNode outputResult = createResultJson(ctx.isPreserveMsgTs(), output.getName(), result); + JsonNode outputResult = createResultJson(ctx.isUseLatestTs(), output.getName(), result); return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), outputResult)); } @@ -83,14 +89,20 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { return TbUtils.toFixed(expressionResult, decimals); } - private JsonNode createResultJson(boolean preserveMsgTs, String outputName, Object result) { + private JsonNode createResultJson(boolean useLatestTs, String outputName, Object result) { ObjectNode valuesNode = JacksonUtil.newObjectNode(); - valuesNode.set(outputName, JacksonUtil.valueToTree(result)); + if (result instanceof Double doubleValue) { + valuesNode.put(outputName, doubleValue); + } else if (result instanceof Integer integerValue) { + valuesNode.put(outputName, integerValue); + } else { + valuesNode.set(outputName, JacksonUtil.valueToTree(result)); + } - long lastTimestamp = getLastUpdateTimestamp(); - if (preserveMsgTs && lastTimestamp != -1) { + long latestTs = getLatestTimestamp(); + if (useLatestTs && latestTs != -1) { ObjectNode resultNode = JacksonUtil.newObjectNode(); - resultNode.put("ts", lastTimestamp); + resultNode.put("ts", latestTs); resultNode.set("values", valuesNode); return resultNode; } else { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 3b858e81b2..1585c9b2a9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -107,7 +107,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { @Override public boolean updateEntry(ArgumentEntry entry) { if (entry instanceof SingleValueArgumentEntry singleValueEntry) { - if (singleValueEntry.getTs() == this.ts) { + if (singleValueEntry.getTs() <= this.ts) { return false; } diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java index 173ba742af..0778d61ee7 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.msg.TbMsgType; @@ -62,8 +63,6 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; -import java.util.Collections; -import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.regex.Matcher; @@ -240,10 +239,11 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService { return deviceCredentialsService.updateDeviceCredentials(tenantId, deviceCredentials); } - private ListenableFuture> saveProvisionStateAttribute(Device device) { - return attributesService.save(device.getTenantId(), device.getId(), AttributeScope.SERVER_SCOPE, - Collections.singletonList(new BaseAttributeKvEntry(new StringDataEntry(DEVICE_PROVISION_STATE, PROVISIONED_STATE), - System.currentTimeMillis()))); + private ListenableFuture saveProvisionStateAttribute(Device device) { + return attributesService.save( + device.getTenantId(), device.getId(), AttributeScope.SERVER_SCOPE, + new BaseAttributeKvEntry(new StringDataEntry(DEVICE_PROVISION_STATE, PROVISIONED_STATE), System.currentTimeMillis()) + ); } private DeviceCredentials getDeviceCredentials(Device device) { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java index 5c95a0e98f..5671ffb2ab 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java @@ -335,6 +335,9 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i Edge edge = edgeGrpcSession.getEdge(); TenantId tenantId = edge.getTenantId(); log.info("[{}][{}] edge [{}] connected successfully.", tenantId, edgeGrpcSession.getSessionId(), edgeId); + if (sessions.containsKey(edgeId)) { + destroySession(sessions.get(edgeId)); + } sessions.put(edgeId, edgeGrpcSession); final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock()); newEventLock.lock(); @@ -503,7 +506,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i } finally { newEventLock.unlock(); } - toRemove.destroy(); + destroySession(toRemove); TenantId tenantId = toRemove.getEdge().getTenantId(); save(tenantId, edgeId, ACTIVITY_STATE, false); long lastDisconnectTs = System.currentTimeMillis(); @@ -516,6 +519,12 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i edgeIdServiceIdCache.evict(edgeId); } + private void destroySession(EdgeGrpcSession session) { + try (session) { + session.destroy(); + } + } + private void save(TenantId tenantId, EdgeId edgeId, String key, long value) { log.debug("[{}][{}] Updating long edge telemetry [{}] [{}]", tenantId, edgeId, key, value); if (persistToTelemetry) { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index e4b78a5ad8..b6ecd848ad 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; @@ -581,10 +582,10 @@ public abstract class EdgeGrpcSession implements Closeable { @Override public void onSuccess(@Nullable Pair newStartTsAndSeqId) { if (newStartTsAndSeqId != null) { - ListenableFuture> updateFuture = updateQueueStartTsAndSeqId(newStartTsAndSeqId); + ListenableFuture updateFuture = updateQueueStartTsAndSeqId(newStartTsAndSeqId); Futures.addCallback(updateFuture, new FutureCallback<>() { @Override - public void onSuccess(@Nullable List list) { + public void onSuccess(@Nullable AttributesSaveResult saveResult) { log.debug("[{}][{}] queue offset was updated [{}]", tenantId, edge.getId(), newStartTsAndSeqId); boolean newEventsAvailable; if (fetcher.isSeqIdNewCycleStarted()) { @@ -645,8 +646,7 @@ public abstract class EdgeGrpcSession implements Closeable { log.trace("[{}][{}] entity message processed [{}]", tenantId, edge.getId(), downlinkMsg); } } - case ATTRIBUTES_UPDATED, POST_ATTRIBUTES, ATTRIBUTES_DELETED, TIMESERIES_UPDATED -> - downlinkMsg = ctx.getTelemetryProcessor().convertTelemetryEventToDownlink(edge, edgeEvent); + case ATTRIBUTES_UPDATED, POST_ATTRIBUTES, ATTRIBUTES_DELETED, TIMESERIES_UPDATED -> downlinkMsg = ctx.getTelemetryProcessor().convertTelemetryEventToDownlink(edge, edgeEvent); default -> log.warn("[{}][{}] Unsupported action type [{}]", tenantId, edge.getId(), edgeEvent.getAction()); } } catch (Exception e) { @@ -722,13 +722,14 @@ public abstract class EdgeGrpcSession implements Closeable { return startSeqId; } - private ListenableFuture> updateQueueStartTsAndSeqId(Pair pair) { + private ListenableFuture updateQueueStartTsAndSeqId(Pair pair) { newStartTs = pair.getFirst(); newStartSeqId = pair.getSecond(); log.trace("[{}] updateQueueStartTsAndSeqId [{}][{}][{}]", sessionId, edge.getId(), newStartTs, newStartSeqId); - List attributes = Arrays.asList( + List attributes = List.of( new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_TS_ATTR_KEY, newStartTs), System.currentTimeMillis()), - new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_SEQ_ID_ATTR_KEY, newStartSeqId), System.currentTimeMillis())); + new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_SEQ_ID_ATTR_KEY, newStartSeqId), System.currentTimeMillis()) + ); return ctx.getAttributesService().save(edge.getTenantId(), edge.getId(), AttributeScope.SERVER_SCOPE, attributes); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index 6fcb02e4bc..4eadcfc5e8 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -219,7 +219,7 @@ public abstract class BaseEdgeProcessor implements EdgeProcessor { case CREDENTIALS_UPDATED: case ASSIGNED_TO_CUSTOMER: case UNASSIGNED_FROM_CUSTOMER: - if (edgeId != null) { + if (edgeId != null && !edgeId.equals(originatorEdgeId)) { return saveEdgeEvent(tenantId, edgeId, type, actionType, entityId, body); } else { return processNotificationToRelatedEdges(tenantId, entityId, type, actionType, originatorEdgeId); diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index e9ef8c5ace..d580175aa0 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -64,6 +64,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; @@ -581,9 +582,9 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { Collections.singletonList(new BasicTsKvEntry(System.currentTimeMillis(), new BooleanDataEntry(key, value))), 0L); addTsCallback(saveFuture, new TelemetrySaveCallback<>(deviceId, key, value)); } else { - ListenableFuture> saveFuture = attributesService.save(TenantId.SYS_TENANT_ID, deviceId, AttributeScope.SERVER_SCOPE, - Collections.singletonList(new BaseAttributeKvEntry(new BooleanDataEntry(key, value) - , System.currentTimeMillis()))); + ListenableFuture saveFuture = attributesService.save( + TenantId.SYS_TENANT_ID, deviceId, AttributeScope.SERVER_SCOPE, new BaseAttributeKvEntry(new BooleanDataEntry(key, value), System.currentTimeMillis()) + ); addTsCallback(saveFuture, new TelemetrySaveCallback<>(deviceId, key, value)); } } @@ -611,7 +612,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { } private void addTsCallback(ListenableFuture saveFuture, final FutureCallback callback) { - Futures.addCallback(saveFuture, new FutureCallback() { + Futures.addCallback(saveFuture, new FutureCallback<>() { @Override public void onSuccess(@Nullable S result) { callback.onSuccess(result); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index a14ba10bfb..9d20b4d74e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -130,11 +130,9 @@ public class DefaultTbClusterService implements TbClusterService { private final AtomicInteger toEdgeNfs = new AtomicInteger(0); @Autowired - @Lazy private PartitionService partitionService; @Autowired - @Lazy private TbQueueProducerProvider producerProvider; @Autowired diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index cc476d377d..23fdc6b8c0 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -27,11 +27,10 @@ import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; -import lombok.Getter; import lombok.RequiredArgsConstructor; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; +import org.checkerframework.checker.nullness.qual.NonNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; @@ -170,35 +169,22 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService> stats = new HashMap<>(); for (DeviceStateData stateData : deviceStates.values()) { @@ -574,7 +559,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService() { + @Override + public void onSuccess(Void success) { + stateData.getState().setLastInactivityAlarmTime(ts); + onDeviceActivityStatusChange(false, stateData); + } + + @Override + public void onFailure(@NonNull Throwable t) { + log.error("[{}][{}] Failed to update device last inactivity alarm time to '{}'. Device state data: {}", tenantId, deviceId, ts, stateData, t); + } + }, deviceStateCallbackExecutor); } - boolean isActive(long ts, DeviceState state) { + private static boolean isActive(long ts, DeviceState state) { return ts < state.getLastActivityTime() + state.getInactivityTimeout(); } @@ -616,17 +611,32 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService() { + @Override + public void onSuccess(Void success) { + stateData.getState().setActive(active); + pushRuleEngineMessage(stateData, active ? TbMsgType.ACTIVITY_EVENT : TbMsgType.INACTIVITY_EVENT); + TbMsgMetaData metaData = stateData.getMetaData(); + notificationRuleProcessor.process(DeviceActivityTrigger.builder() + .tenantId(tenantId) + .customerId(stateData.getCustomerId()) + .deviceId(deviceId) + .active(active) + .deviceName(metaData.getValue("deviceName")) + .deviceType(metaData.getValue("deviceType")) + .deviceLabel(metaData.getValue("deviceLabel")) + .build()); + } + + @Override + public void onFailure(@NonNull Throwable t) { + log.error("[{}][{}] Failed to change device activity status to '{}'. Device state data: {}", tenantId, deviceId, active, stateData, t); + } + }, deviceStateCallbackExecutor); } boolean cleanDeviceStateIfBelongsToExternalPartition(TenantId tenantId, final DeviceId deviceId) { @@ -634,8 +644,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService save(TenantId tenantId, DeviceId deviceId, String key, long value) { + return save(tenantId, deviceId, new LongDataEntry(key, value), getCurrentTimeMillis()); } - private void save(TenantId tenantId, DeviceId deviceId, String key, boolean value) { - save(tenantId, deviceId, new BooleanDataEntry(key, value), getCurrentTimeMillis()); + private ListenableFuture save(TenantId tenantId, DeviceId deviceId, String key, boolean value) { + return save(tenantId, deviceId, new BooleanDataEntry(key, value), getCurrentTimeMillis()); } - private void save(TenantId tenantId, DeviceId deviceId, KvEntry kvEntry, long ts) { + private ListenableFuture save(TenantId tenantId, DeviceId deviceId, KvEntry kvEntry, long ts) { + ListenableFuture future; if (persistToTelemetry) { - tsSubService.saveTimeseriesInternal(TimeseriesSaveRequest.builder() + future = tsSubService.saveTimeseriesInternal(TimeseriesSaveRequest.builder() .tenantId(tenantId) .entityId(deviceId) .entry(new BasicTsKvEntry(ts, kvEntry)) @@ -895,7 +908,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService(deviceId, kvEntry)) .build()); } else { - tsSubService.saveAttributes(AttributesSaveRequest.builder() + future = tsSubService.saveAttributesInternal(AttributesSaveRequest.builder() .tenantId(tenantId) .entityId(deviceId) .scope(AttributeScope.SERVER_SCOPE) @@ -903,20 +916,14 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService(deviceId, kvEntry)) .build()); } + return Futures.transform(future, __ -> null, MoreExecutors.directExecutor()); } long getCurrentTimeMillis() { return System.currentTimeMillis(); } - private static class TelemetrySaveCallback implements FutureCallback { - private final DeviceId deviceId; - private final KvEntry kvEntry; - - TelemetrySaveCallback(DeviceId deviceId, KvEntry kvEntry) { - this.deviceId = deviceId; - this.kvEntry = kvEntry; - } + private record TelemetrySaveCallback(DeviceId deviceId, KvEntry kvEntry) implements FutureCallback { @Override public void onSuccess(@Nullable T result) { @@ -924,9 +931,10 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService saveAttributesInternal(AttributesSaveRequest request) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); AttributesSaveRequest.Strategy strategy = request.getStrategy(); - ListenableFuture> resultFuture; + ListenableFuture resultFuture; if (strategy.saveAttributes()) { resultFuture = attrService.save(tenantId, entityId, request.getScope(), request.getEntries()); } else { - resultFuture = Futures.immediateFuture(Collections.emptyList()); + resultFuture = Futures.immediateFuture(AttributesSaveResult.EMPTY); } addMainCallback(resultFuture, result -> { @@ -228,6 +228,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (strategy.sendWsUpdate()) { addWsCallback(resultFuture, success -> onAttributesUpdate(tenantId, entityId, request.getScope().name(), request.getEntries())); } + return resultFuture; } private static boolean shouldSendSharedAttributesUpdatedNotification(AttributesSaveRequest request) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java index 8a76aa1d14..2d3be5a0ba 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java @@ -21,6 +21,7 @@ import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; /** @@ -30,7 +31,7 @@ public interface InternalTelemetryService extends RuleEngineTelemetryService { ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request); - void saveAttributesInternal(AttributesSaveRequest request); + ListenableFuture saveAttributesInternal(AttributesSaveRequest request); void deleteTimeseriesInternal(TimeseriesDeleteRequest request); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 6d27b9bd7e..c3d28f9fce 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -346,6 +346,7 @@ sql: stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}" # Interval in milliseconds for printing timeseries insert statistic batch_threads: "${SQL_TS_BATCH_THREADS:3}" # batch thread count has to be a prime number like 3 or 5 to gain perfect hash distribution value_no_xss_validation: "${SQL_TS_VALUE_NO_XSS_VALIDATION:false}" # If true telemetry values will be checked for XSS vulnerability + callback_thread_pool_size: "${SQL_TS_CALLBACK_THREAD_POOL_SIZE:12}" # Thread pool size for telemetry callback executor ts_latest: batch_size: "${SQL_TS_LATEST_BATCH_SIZE:1000}" # Batch size for persisting latest telemetry updates batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:50}" # Maximum timeout for latest telemetry entries queue polling. The value set in milliseconds @@ -898,6 +899,8 @@ state: # Used only when state.persistToTelemetry is set to 'true' and Cassandra is used for timeseries data. # 0 means time-to-live mechanism is disabled. telemetryTtl: "${STATE_TELEMETRY_TTL:0}" + # Number of device records to fetch per batch when initializing device activity states + initFetchPackSize: "${TB_DEVICE_STATE_INIT_FETCH_PACK_SIZE:50000}" # Configuration properties for rule nodes related to device activity state rule: node: diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index f65f6bc629..c8b8b0244b 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -464,7 +464,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes } @Test - public void testSimpleCalculatedFieldWhenPreserveMsgTsIsTrue() throws Exception { + public void testSimpleCalculatedFieldWhenUseLatestTsIsTrue() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); long ts = System.currentTimeMillis() - 300000L; doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"temperature\":30}}", ts))); @@ -489,7 +489,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes output.setType(OutputType.TIME_SERIES); config.setOutput(output); - config.setPreserveMsgTs(true); + config.setUseLatestTs(true); calculatedField.setConfiguration(config); @@ -506,7 +506,69 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes } @Test - public void testScriptCalculatedFieldWhenUsedMsgTsInScript() throws Exception { + public void testSimpleCalculatedFieldWhenUseLatestTsIsTrueAndTelemetryBeforeLatest() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + long ts = System.currentTimeMillis(); + + long tsA = ts - 300000L; + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"a\":1}}", tsA))); + + long tsB = ts - 300L; + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"b\":5}}", tsB))); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("a + b"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument1 = new Argument(); + ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("a", ArgumentType.TS_LATEST, null); + argument1.setRefEntityKey(refEntityKey1); + Argument argument2 = new Argument(); + ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("b", ArgumentType.TS_LATEST, null); + argument2.setRefEntityKey(refEntityKey2); + config.setArguments(Map.of("a", argument1, "b", argument2)); + config.setExpression("a + b"); + + Output output = new Output(); + output.setName("c"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + config.setUseLatestTs(true); + + calculatedField.setConfiguration(config); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + await().alias("create CF -> perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode c = getLatestTelemetry(testDevice.getId(), "c"); + assertThat(c).isNotNull(); + assertThat(c.get("c").get(0).get("ts").asText()).isEqualTo(Long.toString(tsB)); + assertThat(c.get("c").get(0).get("value").asText()).isEqualTo("6.0"); + }); + + long tsABeforeTsB = tsB - 300L; + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"a\":10}}", tsABeforeTsB))); + + await().alias("update telemetry with ts less than latest -> save result with latest ts").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode c = getLatestTelemetry(testDevice.getId(), "c"); + assertThat(c).isNotNull(); + assertThat(c.get("c").get(0).get("ts").asText()).isEqualTo(Long.toString(tsB));// also tsB, since this is the latest timestamp + assertThat(c.get("c").get(0).get("value").asText()).isEqualTo("15.0"); + }); + } + + @Test + public void testScriptCalculatedFieldWhenUsedLatestTsInScript() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); long ts = System.currentTimeMillis() - 300000L; doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"temperature\":30}}", ts))); @@ -524,7 +586,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); - config.setExpression("return {\"ts\": ctx.msgTs, \"values\": {\"fahrenheitTemp\": (T * 1.8) + 32}};"); + config.setExpression("return {\"ts\": ctx.latestTs, \"values\": {\"fahrenheitTemp\": (T * 1.8) + 32}};"); Output output = new Output(); output.setType(OutputType.TIME_SERIES); diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index f0059e4f6f..c1616a85db 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; @@ -143,7 +144,7 @@ public class SimpleCalculatedFieldStateTest { } @Test - void testPerformCalculationWhenPassedNotNumber() { + void testPerformCalculationWhenPassedString() { state.arguments = new HashMap<>(Map.of( "key1", key1ArgEntry, "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 9, new StringDataEntry("key2", "string"), 124L), @@ -155,6 +156,23 @@ public class SimpleCalculatedFieldStateTest { .hasMessage("Argument 'key2' is not a number."); } + @Test + void testPerformCalculationWhenPassedBoolean() throws ExecutionException, InterruptedException { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 9, new BooleanDataEntry("key2", true), 124L),// true is parsed as 1 + "key3", key3ArgEntry + )); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + Output output = getCalculatedFieldConfig().getOutput(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 35))); + } + @Test void testPerformCalculationWhenDecimalsByDefault() throws ExecutionException, InterruptedException { state.arguments = new HashMap<>(Map.of( diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java index 50cac8a6fe..5ea3808468 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -53,7 +53,7 @@ public class SingleValueArgumentEntryTest { } @Test - void testUpdateEntryWithThaSameTs() { + void testUpdateEntryWithTheSameTs() { assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts, new LongDataEntry("key", 13L), 363L))).isFalse(); } @@ -81,6 +81,11 @@ public class SingleValueArgumentEntryTest { assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 11L), 364L))).isTrue(); } + @Test + void testUpdateEntryWithOldTs() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts - 10, new LongDataEntry("key", 14L), 365L))).isFalse(); + } + @Test void testToTbelCfArgWhenJsonIsObject() { entry = new SingleValueArgumentEntry(ts, new JsonDataEntry("key", "{\"test\": 10}"), 370L); diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index 264ac70443..d8bcd42bc8 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.IdBased; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; @@ -395,7 +396,7 @@ public class EntityServiceTest extends AbstractControllerTest { List highTemperatures = new ArrayList<>(); createTestHierarchy(tenantId, assets, devices, new ArrayList<>(), new ArrayList<>(), temperatures, highTemperatures); - List>> attributeFutures = new ArrayList<>(); + List> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), AttributeScope.CLIENT_SCOPE)); @@ -545,7 +546,7 @@ public class EntityServiceTest extends AbstractControllerTest { List highTemperatures = new ArrayList<>(); createTestHierarchy(tenantId, assets, devices, new ArrayList<>(), new ArrayList<>(), temperatures, highTemperatures); - List>> attributeFutures = new ArrayList<>(); + List> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), AttributeScope.CLIENT_SCOPE)); @@ -599,7 +600,7 @@ public class EntityServiceTest extends AbstractControllerTest { List highConsumptions = new ArrayList<>(); createTestHierarchy(tenantId, assets, devices, consumptions, highConsumptions, new ArrayList<>(), new ArrayList<>()); - List>> attributeFutures = new ArrayList<>(); + List> attributeFutures = new ArrayList<>(); for (int i = 0; i < assets.size(); i++) { Asset asset = assets.get(i); attributeFutures.add(saveLongAttribute(asset.getId(), "consumption", consumptions.get(i), AttributeScope.SERVER_SCOPE)); @@ -1506,7 +1507,7 @@ public class EntityServiceTest extends AbstractControllerTest { } } - List>> attributeFutures = new ArrayList<>(); + List> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); for (AttributeScope currentScope : AttributeScope.values()) { @@ -1578,7 +1579,7 @@ public class EntityServiceTest extends AbstractControllerTest { } } - List>> attributeFutures = new ArrayList<>(); + List> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), AttributeScope.CLIENT_SCOPE)); @@ -1808,7 +1809,7 @@ public class EntityServiceTest extends AbstractControllerTest { } } - List>> attributeFutures = new ArrayList<>(); + List> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); attributeFutures.add(saveStringAttribute(device.getId(), "attributeString", attributeStrings.get(i), AttributeScope.CLIENT_SCOPE)); @@ -2269,16 +2270,16 @@ public class EntityServiceTest extends AbstractControllerTest { return filter; } - private ListenableFuture> saveLongAttribute(EntityId entityId, String key, long value, AttributeScope scope) { + private ListenableFuture saveLongAttribute(EntityId entityId, String key, long value, AttributeScope scope) { KvEntry attrValue = new LongDataEntry(key, value); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); - return attributesService.save(tenantId, entityId, scope, Collections.singletonList(attr)); + return attributesService.save(tenantId, entityId, scope, List.of(attr)); } - private ListenableFuture> saveStringAttribute(EntityId entityId, String key, String value, AttributeScope scope) { + private ListenableFuture saveStringAttribute(EntityId entityId, String key, String value, AttributeScope scope) { KvEntry attrValue = new StringDataEntry(key, value); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); - return attributesService.save(tenantId, entityId, scope, Collections.singletonList(attr)); + return attributesService.save(tenantId, entityId, scope, List.of(attr)); } private ListenableFuture saveTimeseries(EntityId entityId, String key, Double value) { @@ -2294,8 +2295,8 @@ public class EntityServiceTest extends AbstractControllerTest { } protected void createMultiRootHierarchy(List buildings, List apartments, - Map> entityNameByTypeMap, - Map childParentRelationMap) throws InterruptedException { + Map> entityNameByTypeMap, + Map childParentRelationMap) throws InterruptedException { for (int k = 0; k < 3; k++) { Asset building = new Asset(); building.setTenantId(tenantId); diff --git a/application/src/test/java/org/thingsboard/server/service/job/JobManagerTest.java b/application/src/test/java/org/thingsboard/server/service/job/JobManagerTest.java index 8da1be43f1..0069310b5d 100644 --- a/application/src/test/java/org/thingsboard/server/service/job/JobManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/job/JobManagerTest.java @@ -165,8 +165,7 @@ public class JobManagerTest extends AbstractControllerTest { await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { Job job = findJobById(jobId); assertThat(job.getStatus()).isEqualTo(JobStatus.CANCELLED); - assertThat(job.getResult().getSuccessfulCount()).isBetween(1, tasksCount - 1); - assertThat(job.getResult().getDiscardedCount()).isBetween(1, tasksCount - 1); + assertThat(job.getResult().getDiscardedCount()).isBetween(1, tasksCount); assertThat(job.getResult().getTotalCount()).isEqualTo(tasksCount); assertThat(job.getResult().getCompletedCount()).isEqualTo(tasksCount); }); diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java index dd9c4bebcf..09bd02e5a4 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -467,7 +467,7 @@ public class TbRuleEngineQueueConsumerManagerTest { consumerManager.delete(true); - await().atMost(2, TimeUnit.SECONDS) + await().atMost(5, TimeUnit.SECONDS) .untilAsserted(() -> { verify(ruleEngineMsgProducer).send(any(), any(), any()); }); diff --git a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java index 26e913eacc..5bf87137ff 100644 --- a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java @@ -16,6 +16,9 @@ package org.thingsboard.server.service.state; import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -31,13 +34,13 @@ import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceIdInfo; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.notification.rule.trigger.DeviceActivityTrigger; -import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; @@ -50,20 +53,22 @@ import org.thingsboard.server.dao.sql.query.EntityQueryRepository; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.QueueKey; -import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.usagestats.DefaultTbApiUsageReportClient; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; +import java.time.Duration; import java.util.Collections; +import java.util.HashSet; import java.util.List; -import java.util.Map; +import java.util.Set; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -77,8 +82,8 @@ import static org.mockito.BDDMockito.then; import static org.mockito.BDDMockito.willReturn; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.never; -import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -90,7 +95,10 @@ import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAS import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_DISCONNECT_TIME; @ExtendWith(MockitoExtension.class) -public class DefaultDeviceStateServiceTest { +class DefaultDeviceStateServiceTest { + + ListeningExecutorService deviceStateExecutor; + ListeningExecutorService deviceStateCallbackExecutor; @Mock DeviceService deviceService; @@ -113,25 +121,48 @@ public class DefaultDeviceStateServiceTest { @Mock DefaultTbApiUsageReportClient defaultTbApiUsageReportClient; - TenantId tenantId = new TenantId(UUID.fromString("00797a3b-7aeb-4b5b-b57a-c2a810d0f112")); - DeviceId deviceId = DeviceId.fromString("00797a3b-7aeb-4b5b-b57a-c2a810d0f112"); - TopicPartitionInfo tpi; + long defaultInactivityTimeoutMs = Duration.ofMinutes(10L).toMillis(); + + TenantId tenantId = TenantId.fromUUID(UUID.fromString("00797a3b-7aeb-4b5b-b57a-c2a810d0f112")); + DeviceId deviceId = DeviceId.fromString("c209f718-42e5-11f0-9fe2-0242ac120002"); + TopicPartitionInfo tpi = TopicPartitionInfo.builder() + .topic("tb_core") + .partition(0) + .myPartition(true) + .build(); DefaultDeviceStateService service; @BeforeEach - public void setUp() { + void setUp() { service = spy(new DefaultDeviceStateService(deviceService, attributesService, tsService, clusterService, partitionService, entityQueryRepository, null, defaultTbApiUsageReportClient, notificationRuleProcessor)); ReflectionTestUtils.setField(service, "tsSubService", telemetrySubscriptionService); + ReflectionTestUtils.setField(service, "defaultInactivityTimeoutMs", defaultInactivityTimeoutMs); ReflectionTestUtils.setField(service, "defaultStateCheckIntervalInSec", 60); ReflectionTestUtils.setField(service, "defaultActivityStatsIntervalInSec", 60); - ReflectionTestUtils.setField(service, "initFetchPackSize", 10); + ReflectionTestUtils.setField(service, "initFetchPackSize", 50000); + + deviceStateExecutor = MoreExecutors.newDirectExecutorService(); + ReflectionTestUtils.setField(service, "deviceStateExecutor", deviceStateExecutor); + + deviceStateCallbackExecutor = MoreExecutors.newDirectExecutorService(); + ReflectionTestUtils.setField(service, "deviceStateCallbackExecutor", deviceStateCallbackExecutor); + + lenient().when(partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId)).thenReturn(tpi); + + ConcurrentMap> partitionedEntities = new ConcurrentHashMap<>(); + partitionedEntities.put(tpi, new HashSet<>()); + ReflectionTestUtils.setField(service, "partitionedEntities", partitionedEntities); + } - tpi = TopicPartitionInfo.builder().myPartition(true).build(); + @AfterEach + void cleanup() { + deviceStateExecutor.shutdownNow(); + deviceStateCallbackExecutor.shutdownNow(); } @Test - public void givenDeviceBelongsToExternalPartition_whenOnDeviceConnect_thenCleansStateAndDoesNotReportConnect() { + void givenDeviceBelongsToExternalPartition_whenOnDeviceConnect_thenCleansStateAndDoesNotReportConnect() { // GIVEN doReturn(true).when(service).cleanDeviceStateIfBelongsToExternalPartition(tenantId, deviceId); @@ -149,7 +180,7 @@ public class DefaultDeviceStateServiceTest { @ParameterizedTest @ValueSource(longs = {Long.MIN_VALUE, -100, -1}) - public void givenNegativeLastConnectTime_whenOnDeviceConnect_thenSkipsThisEvent(long negativeLastConnectTime) { + void givenNegativeLastConnectTime_whenOnDeviceConnect_thenSkipsThisEvent(long negativeLastConnectTime) { // GIVEN doReturn(false).when(service).cleanDeviceStateIfBelongsToExternalPartition(tenantId, deviceId); @@ -166,7 +197,7 @@ public class DefaultDeviceStateServiceTest { @ParameterizedTest @MethodSource("provideOutdatedTimestamps") - public void givenOutdatedLastConnectTime_whenOnDeviceDisconnect_thenSkipsThisEvent(long outdatedLastConnectTime, long currentLastConnectTime) { + void givenOutdatedLastConnectTime_whenOnDeviceDisconnect_thenSkipsThisEvent(long outdatedLastConnectTime, long currentLastConnectTime) { // GIVEN doReturn(false).when(service).cleanDeviceStateIfBelongsToExternalPartition(tenantId, deviceId); @@ -188,7 +219,7 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceBelongsToMyPartition_whenOnDeviceConnect_thenReportsConnect() { + void givenDeviceBelongsToMyPartition_whenOnDeviceConnect_thenReportsConnect() { // GIVEN var deviceStateData = DeviceStateData.builder() .tenantId(tenantId) @@ -202,11 +233,13 @@ public class DefaultDeviceStateServiceTest { service.deviceStates.put(deviceId, deviceStateData); long lastConnectTime = System.currentTimeMillis(); + mockSuccessfulSaveAttributes(); + // WHEN service.onDeviceConnect(tenantId, deviceId, lastConnectTime); // THEN - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(LAST_CONNECT_TIME) && @@ -221,7 +254,7 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceBelongsToExternalPartition_whenOnDeviceDisconnect_thenCleansStateAndDoesNotReportDisconnect() { + void givenDeviceBelongsToExternalPartition_whenOnDeviceDisconnect_thenCleansStateAndDoesNotReportDisconnect() { // GIVEN doReturn(true).when(service).cleanDeviceStateIfBelongsToExternalPartition(tenantId, deviceId); @@ -238,7 +271,7 @@ public class DefaultDeviceStateServiceTest { @ParameterizedTest @ValueSource(longs = {Long.MIN_VALUE, -100, -1}) - public void givenNegativeLastDisconnectTime_whenOnDeviceDisconnect_thenSkipsThisEvent(long negativeLastDisconnectTime) { + void givenNegativeLastDisconnectTime_whenOnDeviceDisconnect_thenSkipsThisEvent(long negativeLastDisconnectTime) { // GIVEN doReturn(false).when(service).cleanDeviceStateIfBelongsToExternalPartition(tenantId, deviceId); @@ -254,7 +287,7 @@ public class DefaultDeviceStateServiceTest { @ParameterizedTest @MethodSource("provideOutdatedTimestamps") - public void givenOutdatedLastDisconnectTime_whenOnDeviceDisconnect_thenSkipsThisEvent(long outdatedLastDisconnectTime, long currentLastDisconnectTime) { + void givenOutdatedLastDisconnectTime_whenOnDeviceDisconnect_thenSkipsThisEvent(long outdatedLastDisconnectTime, long currentLastDisconnectTime) { // GIVEN doReturn(false).when(service).cleanDeviceStateIfBelongsToExternalPartition(tenantId, deviceId); @@ -275,7 +308,7 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceBelongsToMyPartition_whenOnDeviceDisconnect_thenReportsDisconnect() { + void givenDeviceBelongsToMyPartition_whenOnDeviceDisconnect_thenReportsDisconnect() { // GIVEN var deviceStateData = DeviceStateData.builder() .tenantId(tenantId) @@ -289,11 +322,13 @@ public class DefaultDeviceStateServiceTest { service.deviceStates.put(deviceId, deviceStateData); long lastDisconnectTime = System.currentTimeMillis(); + mockSuccessfulSaveAttributes(); + // WHEN service.onDeviceDisconnect(tenantId, deviceId, lastDisconnectTime); // THEN - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(LAST_DISCONNECT_TIME) && @@ -308,7 +343,7 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceBelongsToExternalPartition_whenOnDeviceInactivity_thenCleansStateAndDoesNotReportInactivity() { + void givenDeviceBelongsToExternalPartition_whenOnDeviceInactivity_thenCleansStateAndDoesNotReportInactivity() { // GIVEN doReturn(true).when(service).cleanDeviceStateIfBelongsToExternalPartition(tenantId, deviceId); @@ -325,7 +360,7 @@ public class DefaultDeviceStateServiceTest { @ParameterizedTest @ValueSource(longs = {Long.MIN_VALUE, -100, -1}) - public void givenNegativeLastInactivityTime_whenOnDeviceInactivity_thenSkipsThisEvent(long negativeLastInactivityTime) { + void givenNegativeLastInactivityTime_whenOnDeviceInactivity_thenSkipsThisEvent(long negativeLastInactivityTime) { // GIVEN doReturn(false).when(service).cleanDeviceStateIfBelongsToExternalPartition(tenantId, deviceId); @@ -341,7 +376,7 @@ public class DefaultDeviceStateServiceTest { @ParameterizedTest @MethodSource("provideOutdatedTimestamps") - public void givenReceivedInactivityTimeIsLessThanOrEqualToCurrentInactivityTime_whenOnDeviceInactivity_thenSkipsThisEvent( + void givenReceivedInactivityTimeIsLessThanOrEqualToCurrentInactivityTime_whenOnDeviceInactivity_thenSkipsThisEvent( long outdatedLastInactivityTime, long currentLastInactivityTime ) { // GIVEN @@ -365,7 +400,7 @@ public class DefaultDeviceStateServiceTest { @ParameterizedTest @MethodSource("provideOutdatedTimestamps") - public void givenReceivedInactivityTimeIsLessThanOrEqualToCurrentActivityTime_whenOnDeviceInactivity_thenSkipsThisEvent( + void givenReceivedInactivityTimeIsLessThanOrEqualToCurrentActivityTime_whenOnDeviceInactivity_thenSkipsThisEvent( long outdatedLastInactivityTime, long currentLastActivityTime ) { // GIVEN @@ -398,7 +433,7 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceBelongsToMyPartition_whenOnDeviceInactivity_thenReportsInactivity() { + void givenDeviceBelongsToMyPartition_whenOnDeviceInactivity_thenReportsInactivity() { // GIVEN var deviceStateData = DeviceStateData.builder() .tenantId(tenantId) @@ -412,17 +447,19 @@ public class DefaultDeviceStateServiceTest { service.deviceStates.put(deviceId, deviceStateData); long lastInactivityTime = System.currentTimeMillis(); + mockSuccessfulSaveAttributes(); + // WHEN service.onDeviceInactivity(tenantId, deviceId, lastInactivityTime); // THEN - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) && request.getEntries().get(0).getValue().equals(lastInactivityTime) )); - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && @@ -445,7 +482,7 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenInactivityTimeoutReached_whenUpdateInactivityStateIfExpired_thenReportsInactivity() { + void givenInactivityTimeoutReached_whenUpdateInactivityStateIfExpired_thenReportsInactivity() { // GIVEN var deviceStateData = DeviceStateData.builder() .tenantId(tenantId) @@ -456,16 +493,18 @@ public class DefaultDeviceStateServiceTest { given(partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId)).willReturn(tpi); + mockSuccessfulSaveAttributes(); + // WHEN service.updateInactivityStateIfExpired(System.currentTimeMillis(), deviceId, deviceStateData); // THEN - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) )); - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && @@ -488,7 +527,7 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceIdFromDeviceStatesMap_whenGetOrFetchDeviceStateData_thenNoStackOverflow() { + void givenDeviceIdFromDeviceStatesMap_whenGetOrFetchDeviceStateData_thenNoStackOverflow() { service.deviceStates.put(deviceId, deviceStateDataMock); DeviceStateData deviceStateData = service.getOrFetchDeviceStateData(deviceId); assertThat(deviceStateData).isEqualTo(deviceStateDataMock); @@ -496,7 +535,7 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceIdWithoutDeviceStateInMap_whenGetOrFetchDeviceStateData_thenFetchDeviceStateData() { + void givenDeviceIdWithoutDeviceStateInMap_whenGetOrFetchDeviceStateData_thenFetchDeviceStateData() { service.deviceStates.clear(); willReturn(deviceStateDataMock).given(service).fetchDeviceStateDataUsingSeparateRequests(deviceId); DeviceStateData deviceStateData = service.getOrFetchDeviceStateData(deviceId); @@ -504,172 +543,18 @@ public class DefaultDeviceStateServiceTest { verify(service).fetchDeviceStateDataUsingSeparateRequests(deviceId); } - private void initStateService(long timeout) throws InterruptedException { - service.stop(); - reset(service, telemetrySubscriptionService); - service.setDefaultInactivityTimeoutMs(timeout); - service.init(); - when(partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId)).thenReturn(tpi); - when(entityQueryRepository.findEntityDataByQueryInternal(any())).thenReturn(new PageData<>()); - var deviceIdInfo = new DeviceIdInfo(tenantId.getId(), null, deviceId.getId()); - when(deviceService.findDeviceIdInfos(any())) - .thenReturn(new PageData<>(List.of(deviceIdInfo), 0, 1, false)); - PartitionChangeEvent event = new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of( - new QueueKey(ServiceType.TB_CORE), Collections.singleton(tpi) - ), Collections.emptyMap()); - service.onApplicationEvent(event); - Thread.sleep(100); - } - - @Test - public void increaseInactivityForInactiveDeviceTest() throws Exception { - final long defaultTimeout = 1; - initStateService(defaultTimeout); - DeviceState deviceState = DeviceState.builder().build(); - DeviceStateData deviceStateData = DeviceStateData.builder() - .tenantId(tenantId) - .deviceId(deviceId) - .state(deviceState) - .metaData(new TbMsgMetaData()) - .build(); - - service.deviceStates.put(deviceId, deviceStateData); - service.getPartitionedEntities(tpi).add(deviceId); - - service.onDeviceActivity(tenantId, deviceId, System.currentTimeMillis()); - activityVerify(true); - Thread.sleep(defaultTimeout); - service.checkStates(); - activityVerify(false); - - reset(telemetrySubscriptionService); - - long increase = 100; - long newTimeout = System.currentTimeMillis() - deviceState.getLastActivityTime() + increase; - - service.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, newTimeout); - activityVerify(true); - Thread.sleep(increase); - service.checkStates(); - activityVerify(false); - - reset(telemetrySubscriptionService); - - service.onDeviceActivity(tenantId, deviceId, System.currentTimeMillis()); - activityVerify(true); - Thread.sleep(newTimeout + 5); - service.checkStates(); - activityVerify(false); - } - - @Test - public void increaseInactivityForActiveDeviceTest() throws Exception { - final long defaultTimeout = 1000; - initStateService(defaultTimeout); - DeviceState deviceState = DeviceState.builder().build(); - DeviceStateData deviceStateData = DeviceStateData.builder() - .tenantId(tenantId) - .deviceId(deviceId) - .state(deviceState) - .metaData(new TbMsgMetaData()) - .build(); - - service.deviceStates.put(deviceId, deviceStateData); - service.getPartitionedEntities(tpi).add(deviceId); - - service.onDeviceActivity(tenantId, deviceId, System.currentTimeMillis()); - activityVerify(true); - - reset(telemetrySubscriptionService); - - long increase = 100; - long newTimeout = System.currentTimeMillis() - deviceState.getLastActivityTime() + increase; - - service.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, newTimeout); - verify(telemetrySubscriptionService, never()).saveAttributes(argThat(request -> - request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) - )); - Thread.sleep(defaultTimeout + increase); - service.checkStates(); - activityVerify(false); - - reset(telemetrySubscriptionService); - - service.onDeviceActivity(tenantId, deviceId, System.currentTimeMillis()); - activityVerify(true); - Thread.sleep(newTimeout); - service.checkStates(); - activityVerify(false); - } - - @Test - public void increaseSmallInactivityForInactiveDeviceTest() throws Exception { - final long defaultTimeout = 1; - initStateService(defaultTimeout); - DeviceState deviceState = DeviceState.builder().build(); - DeviceStateData deviceStateData = DeviceStateData.builder() - .tenantId(tenantId) - .deviceId(deviceId) - .state(deviceState) - .metaData(new TbMsgMetaData()) - .build(); - - service.deviceStates.put(deviceId, deviceStateData); - service.getPartitionedEntities(tpi).add(deviceId); - - service.onDeviceActivity(tenantId, deviceId, System.currentTimeMillis()); - activityVerify(true); - Thread.sleep(defaultTimeout); - service.checkStates(); - activityVerify(false); - - reset(telemetrySubscriptionService); - - long newTimeout = 1; - Thread.sleep(newTimeout); - verify(telemetrySubscriptionService, never()).saveAttributes(argThat(request -> - request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) - )); - } + @MethodSource + @ParameterizedTest + void testOnDeviceInactivityTimeoutUpdate(boolean initialActivityStatus, long newInactivityTimeout, boolean expectedActivityStatus) { + // GIVEN + doReturn(200L).when(service).getCurrentTimeMillis(); - @Test - public void decreaseInactivityForActiveDeviceTest() throws Exception { - final long defaultTimeout = 1000; - initStateService(defaultTimeout); - DeviceState deviceState = DeviceState.builder().build(); - DeviceStateData deviceStateData = DeviceStateData.builder() - .tenantId(tenantId) - .deviceId(deviceId) - .state(deviceState) - .metaData(new TbMsgMetaData()) + var deviceState = DeviceState.builder() + .active(initialActivityStatus) + .lastActivityTime(100L) .build(); - service.deviceStates.put(deviceId, deviceStateData); - service.getPartitionedEntities(tpi).add(deviceId); - - service.onDeviceActivity(tenantId, deviceId, System.currentTimeMillis()); - activityVerify(true); - - long newTimeout = 1; - Thread.sleep(newTimeout); - - service.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, newTimeout); - activityVerify(false); - reset(telemetrySubscriptionService); - - service.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, defaultTimeout); - activityVerify(true); - Thread.sleep(defaultTimeout); - service.checkStates(); - activityVerify(false); - } - - @Test - public void decreaseInactivityForInactiveDeviceTest() throws Exception { - final long defaultTimeout = 1000; - initStateService(defaultTimeout); - DeviceState deviceState = DeviceState.builder().build(); - DeviceStateData deviceStateData = DeviceStateData.builder() + var deviceStateData = DeviceStateData.builder() .tenantId(tenantId) .deviceId(deviceId) .state(deviceState) @@ -679,31 +564,44 @@ public class DefaultDeviceStateServiceTest { service.deviceStates.put(deviceId, deviceStateData); service.getPartitionedEntities(tpi).add(deviceId); - service.onDeviceActivity(tenantId, deviceId, System.currentTimeMillis()); - activityVerify(true); - Thread.sleep(defaultTimeout); - service.checkStates(); - activityVerify(false); - reset(telemetrySubscriptionService); + mockSuccessfulSaveAttributes(); - long newTimeout = 1; + // WHEN + service.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, newInactivityTimeout); - service.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, newTimeout); - verify(telemetrySubscriptionService, never()).saveAttributes(argThat(request -> - request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) - )); + // THEN + long expectedInactivityTimeout = newInactivityTimeout != 0 ? newInactivityTimeout : defaultInactivityTimeoutMs; + assertThat(deviceState.getInactivityTimeout()).isEqualTo(expectedInactivityTimeout); + + assertThat(deviceState.isActive()).isEqualTo(expectedActivityStatus); + if (initialActivityStatus != expectedActivityStatus) { + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> { + AttributeKvEntry entry = request.getEntries().get(0); + return request.getEntityId().equals(deviceId) && entry.getKey().equals(ACTIVITY_STATE) && entry.getValue().equals(expectedActivityStatus); + })); + } } - private void activityVerify(boolean isActive) { - verify(telemetrySubscriptionService).saveAttributes(argThat(request -> - request.getEntityId().equals(deviceId) && - request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && - request.getEntries().get(0).getValue().equals(isActive) - )); + // to simplify test, these arguments assume that the current time is 200 and the last activity time is 100 + private static Stream testOnDeviceInactivityTimeoutUpdate() { + return Stream.of( + Arguments.of(true, 1L, false), + Arguments.of(true, 50L, false), + Arguments.of(true, 99L, false), + Arguments.of(true, 100L, false), + Arguments.of(true, 101L, true), + Arguments.of(true, 0L, true), // should use default inactivity timeout of 10 minutes + Arguments.of(false, 1L, false), + Arguments.of(false, 50L, false), + Arguments.of(false, 99L, false), + Arguments.of(false, 100L, false), + Arguments.of(false, 101L, true), + Arguments.of(false, 0L, true) // should use default inactivity timeout of 10 minutes + ); } @Test - public void givenStateDataIsNull_whenUpdateActivityState_thenShouldCleanupDevice() { + void givenStateDataIsNull_whenUpdateActivityState_thenShouldCleanupDevice() { // GIVEN service.deviceStates.put(deviceId, deviceStateDataMock); @@ -719,7 +617,7 @@ public class DefaultDeviceStateServiceTest { @ParameterizedTest @MethodSource("provideParametersForUpdateActivityState") - public void givenTestParameters_whenUpdateActivityState_thenShouldBeInTheExpectedStateAndPerformExpectedActions( + void givenTestParameters_whenUpdateActivityState_thenShouldBeInTheExpectedStateAndPerformExpectedActions( boolean activityState, long previousActivityTime, long lastReportedActivity, long inactivityAlarmTime, long expectedInactivityAlarmTime, boolean shouldSetInactivityAlarmTimeToZero, boolean shouldUpdateActivityStateToActive @@ -739,13 +637,15 @@ public class DefaultDeviceStateServiceTest { .metaData(new TbMsgMetaData()) .build(); + mockSuccessfulSaveAttributes(); + // WHEN service.updateActivityState(deviceId, deviceStateData, lastReportedActivity); // THEN assertThat(deviceState.isActive()).isEqualTo(true); assertThat(deviceState.getLastActivityTime()).isEqualTo(lastReportedActivity); - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(LAST_ACTIVITY_TIME) && request.getEntries().get(0).getValue().equals(lastReportedActivity) @@ -753,7 +653,7 @@ public class DefaultDeviceStateServiceTest { assertThat(deviceState.getLastInactivityAlarmTime()).isEqualTo(expectedInactivityAlarmTime); if (shouldSetInactivityAlarmTimeToZero) { - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) && request.getEntries().get(0).getValue().equals(0L) @@ -761,7 +661,7 @@ public class DefaultDeviceStateServiceTest { } if (shouldUpdateActivityStateToActive) { - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getValue().equals(true) @@ -809,59 +709,8 @@ public class DefaultDeviceStateServiceTest { ); } - @ParameterizedTest - @MethodSource("provideParametersForDecreaseInactivityTimeout") - public void givenTestParameters_whenOnDeviceInactivityTimeout_thenShouldBeInTheExpectedStateAndPerformExpectedActions( - boolean activityState, long newInactivityTimeout, long timeIncrement, boolean expectedActivityState - ) throws Exception { - // GIVEN - long defaultInactivityTimeout = 10000; - initStateService(defaultInactivityTimeout); - - var currentTime = new AtomicLong(System.currentTimeMillis()); - - DeviceState deviceState = DeviceState.builder() - .active(activityState) - .lastActivityTime(currentTime.get()) - .inactivityTimeout(defaultInactivityTimeout) - .build(); - - DeviceStateData deviceStateData = DeviceStateData.builder() - .tenantId(tenantId) - .deviceId(deviceId) - .state(deviceState) - .metaData(new TbMsgMetaData()) - .build(); - - service.deviceStates.put(deviceId, deviceStateData); - service.getPartitionedEntities(tpi).add(deviceId); - - given(service.getCurrentTimeMillis()).willReturn(currentTime.addAndGet(timeIncrement)); - - // WHEN - service.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, newInactivityTimeout); - - // THEN - assertThat(deviceState.getInactivityTimeout()).isEqualTo(newInactivityTimeout); - assertThat(deviceState.isActive()).isEqualTo(expectedActivityState); - if (activityState && !expectedActivityState) { - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && - request.getEntries().get(0).getValue().equals(false) - )); - } - } - - private static Stream provideParametersForDecreaseInactivityTimeout() { - return Stream.of( - Arguments.of(true, 1, 0, true), - - Arguments.of(true, 1, 1, false) - ); - } - @Test - public void givenStateDataIsNull_whenUpdateInactivityTimeoutIfExpired_thenShouldCleanupDevice() { + void givenStateDataIsNull_whenUpdateInactivityTimeoutIfExpired_thenShouldCleanupDevice() { // GIVEN service.deviceStates.put(deviceId, deviceStateDataMock); @@ -875,7 +724,7 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenNotMyPartition_whenUpdateInactivityTimeoutIfExpired_thenShouldCleanupDevice() { + void givenNotMyPartition_whenUpdateInactivityTimeoutIfExpired_thenShouldCleanupDevice() { // GIVEN long currentTime = System.currentTimeMillis(); @@ -911,7 +760,7 @@ public class DefaultDeviceStateServiceTest { @ParameterizedTest @MethodSource("provideParametersForUpdateInactivityStateIfExpired") - public void givenTestParameters_whenUpdateInactivityStateIfExpired_thenShouldBeInTheExpectedStateAndPerformExpectedActions( + void givenTestParameters_whenUpdateInactivityStateIfExpired_thenShouldBeInTheExpectedStateAndPerformExpectedActions( boolean activityState, long ts, long lastActivityTime, long lastInactivityAlarmTime, long inactivityTimeout, long deviceCreationTime, boolean expectedActivityState, long expectedLastInactivityAlarmTime, boolean shouldUpdateActivityStateToInactive ) { @@ -933,6 +782,7 @@ public class DefaultDeviceStateServiceTest { if (shouldUpdateActivityStateToInactive) { given(partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId)).willReturn(tpi); + mockSuccessfulSaveAttributes(); } // WHEN @@ -943,7 +793,7 @@ public class DefaultDeviceStateServiceTest { assertThat(state.getLastInactivityAlarmTime()).isEqualTo(expectedLastInactivityAlarmTime); if (shouldUpdateActivityStateToInactive) { - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getValue().equals(false) )); @@ -961,7 +811,7 @@ public class DefaultDeviceStateServiceTest { assertThat(actualNotification.getDeviceId()).isEqualTo(deviceId); assertThat(actualNotification.isActive()).isFalse(); - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) && @@ -1033,7 +883,79 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenConcurrentAccess_whenGetOrFetchDeviceStateData_thenFetchDeviceStateDataInvokedOnce() { + void givenInactiveDevice_whenActivityStatusChangesToActiveButFailedToSaveUpdatedActivityStatus_thenShouldNotUpdateCache2() { + // GIVEN + var deviceState = DeviceState.builder() + .active(false) + .lastActivityTime(100L) + .inactivityTimeout(50L) + .build(); + + var deviceStateData = DeviceStateData.builder() + .tenantId(tenantId) + .deviceId(deviceId) + .state(deviceState) + .metaData(TbMsgMetaData.EMPTY) + .build(); + + service.deviceStates.put(deviceId, deviceStateData); + service.getPartitionedEntities(tpi).add(deviceId); + + // WHEN-THEN + + // simulating short DB outage + given(telemetrySubscriptionService.saveAttributesInternal(any())).willReturn(Futures.immediateFailedFuture(new RuntimeException("failed to save"))); + doReturn(200L).when(service).getCurrentTimeMillis(); + service.onDeviceActivity(tenantId, deviceId, 180L); + assertThat(deviceState.isActive()).isFalse(); // still inactive + + // 10 millis pass... and new activity message it received + + // this time DB save is successful + when(telemetrySubscriptionService.saveAttributesInternal(any())).thenReturn(Futures.immediateFuture(AttributesSaveResult.of(generateRandomVersions(1)))); + doReturn(210L).when(service).getCurrentTimeMillis(); + service.onDeviceActivity(tenantId, deviceId, 190L); + assertThat(deviceState.isActive()).isTrue(); + } + + @Test + void givenActiveDevice_whenActivityStatusChangesToInactiveButFailedToSaveUpdatedActivityStatus_thenShouldNotUpdateCache() { + // GIVEN + var deviceState = DeviceState.builder() + .active(true) + .lastActivityTime(100L) + .inactivityTimeout(50L) + .build(); + + var deviceStateData = DeviceStateData.builder() + .tenantId(tenantId) + .deviceId(deviceId) + .state(deviceState) + .metaData(TbMsgMetaData.EMPTY) + .build(); + + service.deviceStates.put(deviceId, deviceStateData); + service.getPartitionedEntities(tpi).add(deviceId); + + // WHEN-THEN (assuming periodic activity states check is done every 100 millis) + + // simulating short DB outage + given(telemetrySubscriptionService.saveAttributesInternal(any())).willReturn(Futures.immediateFailedFuture(new RuntimeException("failed to save"))); + doReturn(200L).when(service).getCurrentTimeMillis(); + service.checkStates(); + assertThat(deviceState.isActive()).isTrue(); // still active + + // waiting 100 millis... periodic activity states check is triggered again + + // this time DB save is successful + when(telemetrySubscriptionService.saveAttributesInternal(any())).thenReturn(Futures.immediateFuture(AttributesSaveResult.of(generateRandomVersions(1)))); + doReturn(300L).when(service).getCurrentTimeMillis(); + service.checkStates(); + assertThat(deviceState.isActive()).isFalse(); + } + + @Test + void givenConcurrentAccess_whenGetOrFetchDeviceStateData_thenFetchDeviceStateDataInvokedOnce() { doAnswer(invocation -> { Thread.sleep(100); return deviceStateDataMock; @@ -1069,10 +991,8 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceAdded_whenOnQueueMsg_thenShouldCacheAndSaveActivityToFalse() throws InterruptedException { + void givenDeviceAdded_whenOnQueueMsg_thenShouldCacheAndSaveActivityToFalse() { // GIVEN - final long defaultTimeout = 1000; - initStateService(defaultTimeout); given(deviceService.findDeviceById(any(TenantId.class), any(DeviceId.class))).willReturn(new Device(deviceId)); given(attributesService.find(any(TenantId.class), any(EntityId.class), any(AttributeScope.class), anyCollection())).willReturn(Futures.immediateFuture(Collections.emptyList())); @@ -1086,13 +1006,15 @@ public class DefaultDeviceStateServiceTest { .setDeleted(false) .build(); + mockSuccessfulSaveAttributes(); + // WHEN service.onQueueMsg(proto, TbCallback.EMPTY); // THEN await().atMost(1, TimeUnit.SECONDS).untilAsserted(() -> { assertThat(service.deviceStates.get(deviceId).getState().isActive()).isEqualTo(false); - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getValue().equals(false) )); @@ -1100,14 +1022,12 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceActivityEventHappenedAfterAdded_whenOnDeviceActivity_thenShouldCacheAndSaveActivityToTrue() throws InterruptedException { + void givenDeviceActivityEventHappenedAfterAdded_whenOnDeviceActivity_thenShouldCacheAndSaveActivityToTrue() { // GIVEN - final long defaultTimeout = 1000; - initStateService(defaultTimeout); long currentTime = System.currentTimeMillis(); DeviceState deviceState = DeviceState.builder() .active(false) - .inactivityTimeout(service.getDefaultInactivityTimeoutInSec()) + .inactivityTimeout(defaultInactivityTimeoutMs) .build(); DeviceStateData stateData = DeviceStateData.builder() .tenantId(tenantId) @@ -1118,12 +1038,14 @@ public class DefaultDeviceStateServiceTest { .build(); service.deviceStates.put(deviceId, stateData); + mockSuccessfulSaveAttributes(); + // WHEN service.onDeviceActivity(tenantId, deviceId, currentTime); // THEN ArgumentCaptor attributeRequestCaptor = ArgumentCaptor.forClass(AttributesSaveRequest.class); - then(telemetrySubscriptionService).should(times(2)).saveAttributes(attributeRequestCaptor.capture()); + then(telemetrySubscriptionService).should(times(2)).saveAttributesInternal(attributeRequestCaptor.capture()); await().atMost(1, TimeUnit.SECONDS).untilAsserted(() -> { assertThat(service.deviceStates.get(deviceId).getState().isActive()).isEqualTo(true); @@ -1151,15 +1073,14 @@ public class DefaultDeviceStateServiceTest { } @Test - public void givenDeviceActivityEventHappenedBeforeAdded_whenOnQueueMsg_thenShouldSaveActivityStateUsingValueFromCache() throws InterruptedException { + void givenDeviceActivityEventHappenedBeforeAdded_whenOnQueueMsg_thenShouldSaveActivityStateUsingValueFromCache() { // GIVEN - final long defaultTimeout = 1000; - initStateService(defaultTimeout); given(deviceService.findDeviceById(any(TenantId.class), any(DeviceId.class))).willReturn(new Device(deviceId)); given(attributesService.find(any(TenantId.class), any(EntityId.class), any(AttributeScope.class), anyCollection())).willReturn(Futures.immediateFuture(Collections.emptyList())); long currentTime = System.currentTimeMillis(); - DeviceState deviceState = DeviceState.builder() + + var deviceState = DeviceState.builder() .active(true) .lastConnectTime(currentTime - 8000) .lastActivityTime(currentTime - 4000) @@ -1167,16 +1088,20 @@ public class DefaultDeviceStateServiceTest { .lastInactivityAlarmTime(0) .inactivityTimeout(3000) .build(); - DeviceStateData stateData = DeviceStateData.builder() + + var stateData = DeviceStateData.builder() .tenantId(tenantId) .deviceId(deviceId) .deviceCreationTime(currentTime - 10000) .state(deviceState) .build(); + service.deviceStates.put(deviceId, stateData); + mockSuccessfulSaveAttributes(); + // WHEN - TransportProtos.DeviceStateServiceMsgProto proto = TransportProtos.DeviceStateServiceMsgProto.newBuilder() + var proto = TransportProtos.DeviceStateServiceMsgProto.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) @@ -1190,11 +1115,25 @@ public class DefaultDeviceStateServiceTest { // THEN await().atMost(1, TimeUnit.SECONDS).untilAsserted(() -> { assertThat(service.deviceStates.get(deviceId).getState().isActive()).isEqualTo(true); - then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> + then(telemetrySubscriptionService).should().saveAttributesInternal(argThat(request -> request.getEntityId().equals(deviceId) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getValue().equals(true) )); }); } + private void mockSuccessfulSaveAttributes() { + lenient().when(telemetrySubscriptionService.saveAttributesInternal(any())).thenAnswer(invocation -> { + AttributesSaveRequest request = invocation.getArgument(0); + return Futures.immediateFuture(generateRandomVersions(request.getEntries().size())); + }); + } + + private static List generateRandomVersions(int n) { + return ThreadLocalRandom.current() + .longs(n) + .boxed() + .toList(); + } + } diff --git a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java index 2b4d9f38e5..153228a865 100644 --- a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java @@ -48,6 +48,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; @@ -472,7 +473,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(saveAttributes, sendWsUpdate, processCalculatedFields)) .build(); - lenient().when(attrService.save(tenantId, entityId, request.getScope(), request.getEntries())).thenReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + lenient().when(attrService.save(tenantId, entityId, request.getScope(), request.getEntries())) + .thenReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(request.getEntries().size())))); // WHEN telemetryService.saveAttributes(request); @@ -547,7 +549,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, deviceId, request.getScope(), entries)).willReturn(immediateFuture(listOfNNumbers(entries.size()))); + given(attrService.save(tenantId, deviceId, request.getScope(), entries)) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(entries.size())))); // WHEN telemetryService.saveAttributes(request); @@ -581,7 +584,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, nonDeviceId, request.getScope(), entries)).willReturn(immediateFuture(listOfNNumbers(entries.size()))); + given(attrService.save(tenantId, nonDeviceId, request.getScope(), entries)) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(entries.size())))); // WHEN telemetryService.saveAttributes(request); @@ -613,7 +617,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, deviceId, request.getScope(), entries)).willReturn(immediateFuture(listOfNNumbers(entries.size()))); + given(attrService.save(tenantId, deviceId, request.getScope(), entries)) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(entries.size())))); // WHEN telemetryService.saveAttributes(request); @@ -640,7 +645,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, deviceId, request.getScope(), entries)).willReturn(immediateFuture(listOfNNumbers(entries.size()))); + given(attrService.save(tenantId, deviceId, request.getScope(), entries)) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(entries.size())))); // WHEN telemetryService.saveAttributes(request); @@ -715,7 +721,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())).willReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(request.getEntries().size())))); // WHEN telemetryService.saveAttributes(request); @@ -764,7 +771,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, nonDeviceId, request.getScope(), request.getEntries())).willReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + given(attrService.save(tenantId, nonDeviceId, request.getScope(), request.getEntries())) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(request.getEntries().size())))); // WHEN telemetryService.saveAttributes(request); @@ -792,7 +800,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())).willReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(request.getEntries().size())))); // WHEN telemetryService.saveAttributes(request); @@ -815,7 +824,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())).willReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(request.getEntries().size())))); // WHEN telemetryService.saveAttributes(request); @@ -843,7 +853,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())).willReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(request.getEntries().size())))); // WHEN telemetryService.saveAttributes(request); @@ -870,7 +881,8 @@ class DefaultTelemetrySubscriptionServiceTest { .strategy(new AttributesSaveRequest.Strategy(true, false, false)) .build(); - given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())).willReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())) + .willReturn(immediateFuture(AttributesSaveResult.of(listOfNNumbers(request.getEntries().size())))); // WHEN telemetryService.saveAttributes(request); diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java index 4b822e0030..6cd28fa98d 100644 --- a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java @@ -237,8 +237,14 @@ public final class TbActorMailbox implements TbActorCtx { try { ready.set(NOT_READY); actor.destroy(stopReason, cause); - highPriorityMsgs.forEach(msg -> msg.onTbActorStopped(stopReason)); - normalPriorityMsgs.forEach(msg -> msg.onTbActorStopped(stopReason)); + highPriorityMsgs.removeIf(msg -> { + msg.onTbActorStopped(stopReason); + return true; + }); + normalPriorityMsgs.removeIf(msg -> { + msg.onTbActorStopped(stopReason); + return true; + }); } catch (Throwable t) { log.warn("[{}] Failed to destroy actor: ", selfId, t); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java index 718c574c3c..0d5d3dcd13 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import java.util.Collection; import java.util.List; @@ -37,9 +38,9 @@ public interface AttributesService { ListenableFuture> findAll(TenantId tenantId, EntityId entityId, AttributeScope scope); - ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes); + ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes); - ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute); + ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute); ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributeKeys); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java index ea3297e621..6092e65648 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java @@ -32,8 +32,8 @@ public class AdminSettings extends BaseData implements HasTenan @NoXss @Length(fieldName = "key") private String key; - private transient JsonNode jsonValue; - + private JsonNode jsonValue; + public AdminSettings() { super(); } @@ -41,7 +41,7 @@ public class AdminSettings extends BaseData implements HasTenan public AdminSettings(AdminSettingsId id) { super(id); } - + public AdminSettings(AdminSettings adminSettings) { super(adminSettings); this.tenantId = adminSettings.getTenantId(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java b/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java index 10ea83397b..2ac78b04b6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java @@ -25,9 +25,9 @@ public abstract class BaseData extends IdBased implement private static final long serialVersionUID = 5422817607129962637L; public static final ObjectMapper mapper = new ObjectMapper(); - + protected long createdTime; - + public BaseData() { super(); } @@ -35,7 +35,7 @@ public abstract class BaseData extends IdBased implement public BaseData(I id) { super(id); } - + public BaseData(BaseData data) { super(data.getId()); this.createdTime = data.getCreatedTime(); @@ -53,7 +53,7 @@ public abstract class BaseData extends IdBased implement public int hashCode() { final int prime = 31; int result = super.hashCode(); - result = prime * result + (int) (createdTime ^ (createdTime >>> 32)); + result = prime * result + Long.hashCode(createdTime); return result; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index 1deb322d39..b5ec74c2d3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -68,8 +68,8 @@ public class Dashboard extends DashboardInfo implements ExportableEntity versions) { + + public static final AttributesSaveResult EMPTY = new AttributesSaveResult(Collections.emptyList()); + + public static AttributesSaveResult of(List versions) { + if (versions == null) { + return EMPTY; + } + return new AttributesSaveResult(versions); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java index d9eff103a7..24d5c20d45 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java @@ -62,9 +62,7 @@ public class RuleNode extends BaseDataWithAdditionalInfo implements @Schema(description = "Version of rule node configuration. ", example = "0") private int configurationVersion; @Schema(description = "JSON with the rule node configuration. Structure depends on the rule node implementation.", implementation = JsonNode.class) - private transient JsonNode configuration; - @JsonIgnore - private byte[] configurationBytes; + private JsonNode configuration; private RuleNodeId externalId; @@ -92,14 +90,6 @@ public class RuleNode extends BaseDataWithAdditionalInfo implements return name; } - public JsonNode getConfiguration() { - return BaseDataWithAdditionalInfo.getJson(() -> configuration, () -> configurationBytes); - } - - public void setConfiguration(JsonNode data) { - setJson(data, json -> this.configuration = json, bytes -> this.configurationBytes = bytes); - } - @Schema(description = "JSON object with the Rule Node Id. " + "Specify this field to update the Rule Node. " + "Referencing non-existing Rule Node Id will cause error. " + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/settings/UserSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/settings/UserSettings.java index 5c9ef8fd1f..733d7fc041 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/settings/UserSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/settings/UserSettings.java @@ -15,20 +15,15 @@ */ package org.thingsboard.server.common.data.settings; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; -import lombok.ToString; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; import java.io.Serializable; -import static org.thingsboard.server.common.data.BaseDataWithAdditionalInfo.getJson; -import static org.thingsboard.server.common.data.BaseDataWithAdditionalInfo.setJson; - @Schema @Data public class UserSettings implements Serializable { @@ -43,20 +38,9 @@ public class UserSettings implements Serializable { @Length(fieldName = "type", max = 50) private UserSettingsType type; - @Schema(description = "JSON object with user settings.",implementation = com.fasterxml.jackson.databind.JsonNode.class) + @Schema(description = "JSON object with user settings.", implementation = com.fasterxml.jackson.databind.JsonNode.class) @NoXss @Length(fieldName = "settings", max = 100000) - private transient JsonNode settings; - - @JsonIgnore - @ToString.Exclude - private byte[] settingsBytes; - - public JsonNode getSettings() { - return getJson(() -> settings, () -> settingsBytes); - } + private JsonNode settings; - public void setSettings(JsonNode settings) { - setJson(settings, json -> this.settings = json, bytes -> this.settingsBytes = bytes); - } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetType.java b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetType.java index bdac430ce1..5385a54f74 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetType.java @@ -21,15 +21,17 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.id.WidgetTypeId; import java.util.Optional; +@EqualsAndHashCode(callSuper = true) @Data public class WidgetType extends BaseWidgetType { @Schema(description = "Complex JSON object that describes the widget type", accessMode = Schema.AccessMode.READ_ONLY) - private transient JsonNode descriptor; + private JsonNode descriptor; public WidgetType() { super(); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java index b78e49879e..95a1a79392 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java @@ -36,7 +36,7 @@ public abstract class AbstractEntityProfileNameQueryProcessor(getProfileNames(this.filter)); - pattern = RepositoryUtils.toSqlLikePattern(getEntityNameFilter(filter)); + pattern = RepositoryUtils.toEntityNameSqlLikePattern(getEntityNameFilter(filter)); } protected abstract String getEntityNameFilter(T filter); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java index 301ead7c63..94a7a95404 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java @@ -43,7 +43,7 @@ public abstract class AbstractEntityProfileQueryProcessor ext private Set> getEntitiesSet(RelationsRepo relations) { Set> result = new HashSet<>(); - Set processed = new HashSet<>(); + Set processed = new HashSet<>(); Queue tasks = new LinkedList<>(); int maxLvl = getMaxLevel() == 0 ? MAXIMUM_QUERY_LEVEL : Math.max(1, getMaxLevel()); for (UUID uuid : getRootEntities()) { @@ -114,7 +115,7 @@ public abstract class AbstractRelationQueryProcessor ext } while (!tasks.isEmpty()) { RelationSearchTask task = tasks.poll(); - if (processed.add(task.entityId)) { + if (processed.add(task)) { var entityLvl = task.lvl + 1; Set entities = EntitySearchDirection.FROM.equals(getDirection()) ? relations.getFrom(task.entityId) : relations.getTo(task.entityId); if (isFetchLastLevelOnly() && entities.isEmpty() && task.previous != null && check(task.previous)) { @@ -157,6 +158,7 @@ public abstract class AbstractRelationQueryProcessor ext protected abstract boolean check(RelationInfo relationInfo); @RequiredArgsConstructor + @EqualsAndHashCode private static class RelationSearchTask { private final UUID entityId; private final int lvl; diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java index fbb36cf2a0..879ed9c7b4 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java @@ -30,7 +30,7 @@ public class EntityNameQueryProcessor extends AbstractSimpleQueryProcessor value.equals(predicateValue); - case STARTS_WITH -> value.startsWith(predicateValue); - case ENDS_WITH -> value.endsWith(predicateValue); + case STARTS_WITH -> toSqlLikePattern(predicateValue, "^", ".*").matcher(value).matches(); + case ENDS_WITH -> toSqlLikePattern(predicateValue, ".*", "$").matcher(value).matches(); case NOT_EQUAL -> !value.equals(predicateValue); - case CONTAINS -> value.contains(predicateValue); - case NOT_CONTAINS -> !value.contains(predicateValue); + case CONTAINS -> toSqlLikePattern(predicateValue, ".*", ".*").matcher(value).matches(); + case NOT_CONTAINS -> !toSqlLikePattern(predicateValue, ".*", ".*").matcher(value).matches(); case IN -> equalsAny(value, splitByCommaWithoutQuotes(predicateValue)); case NOT_IN -> !equalsAny(value, splitByCommaWithoutQuotes(predicateValue)); }; @@ -304,6 +303,15 @@ public class RepositoryUtils { return true; } else if (filterPredicates.getOperation() == OR) { for (KeyFilterPredicate filterPredicate : filterPredicates.getPredicates()) { + + // Emulate the SQL-like behavior of ThingsBoard's Entity Data Query service: + // for COMPLEX filters, return no results if filter value is empty + if (filterPredicate instanceof StringFilterPredicate stringFilterPredicate) { + if (StringUtils.isEmpty(stringFilterPredicate.getValue().getValue())) { + continue; + } + } + if (simpleKeyFilter.check(value, filterPredicate)) { return true; } @@ -314,25 +322,34 @@ public class RepositoryUtils { } } - public static Pattern toSqlLikePattern(String nameFilter) { - if (StringUtils.isNotBlank(nameFilter)) { - boolean percentSymbolOnStart = nameFilter.startsWith("%"); - boolean percentSymbolOnEnd = nameFilter.endsWith("%"); - if (percentSymbolOnStart) { - nameFilter = nameFilter.substring(1); - } - if (percentSymbolOnEnd) { - nameFilter = nameFilter.substring(0, nameFilter.length() - 1); - } - if (percentSymbolOnStart || percentSymbolOnEnd) { - return Pattern.compile((percentSymbolOnStart ? ".*" : "") + Pattern.quote(nameFilter) + (percentSymbolOnEnd ? ".*" : ""), Pattern.CASE_INSENSITIVE); - } else { - return Pattern.compile(Pattern.quote(nameFilter) + ".*", Pattern.CASE_INSENSITIVE); - } + public static Pattern toEntityNameSqlLikePattern(String filter) { + if (StringUtils.isNotBlank(filter)) { + return toSqlLikePattern(filter, "", ".*", true); } return null; } + private static Pattern toSqlLikePattern(String value, String prefix, String suffix) { + return toSqlLikePattern(value, prefix, suffix, false); + } + + private static Pattern toSqlLikePattern(String value, String prefix, String suffix, boolean ignoreCase) { + String regexValue; + if (value.contains("%") || value.contains("_")) { + regexValue = value + .replace("_", ".") + .replace("%", ".*"); + if ("^".equals(prefix)) { + regexValue = "^" + regexValue + (regexValue.endsWith(".*") ? "" : ".*"); + } else if ("$".equals(suffix)) { + regexValue = (regexValue.startsWith(".*") ? "" : ".*") + regexValue + "$"; + } + } else { + regexValue = prefix + Pattern.quote(value) + suffix; + } + return ignoreCase ? Pattern.compile(regexValue, Pattern.CASE_INSENSITIVE) : Pattern.compile(regexValue); + } + @FunctionalInterface public interface SimpleKeyFilter { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfCtx.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfCtx.java index 7515cb5269..c6023154ea 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfCtx.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfCtx.java @@ -25,11 +25,11 @@ public class TbelCfCtx implements TbelCfObject { @Getter private final Map args; @Getter - private final long msgTs; + private final long latestTs; - public TbelCfCtx(Map args, long lastUpdateTs) { + public TbelCfCtx(Map args, long latestTs) { this.args = Collections.unmodifiableMap(args); - this.msgTs = lastUpdateTs != -1 ? lastUpdateTs : System.currentTimeMillis(); + this.latestTs = latestTs != -1 ? latestTs : System.currentTimeMillis(); } @Override diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/KeystoreSslCredentials.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/KeystoreSslCredentials.java index edb1e4694b..97f438a8e1 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/KeystoreSslCredentials.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/KeystoreSslCredentials.java @@ -26,7 +26,7 @@ import java.security.GeneralSecurityException; import java.security.KeyStore; @Data -@EqualsAndHashCode(callSuper = false) +@EqualsAndHashCode(callSuper = true) public class KeystoreSslCredentials extends AbstractSslCredentials { private String type; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/PemSslCredentials.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/PemSslCredentials.java index 6c7e0cf842..4d269f6964 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/PemSslCredentials.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/PemSslCredentials.java @@ -46,7 +46,7 @@ import java.util.List; import java.util.stream.Collectors; @Data -@EqualsAndHashCode(callSuper = false) +@EqualsAndHashCode(callSuper = true) public class PemSslCredentials extends AbstractSslCredentials { private static final String DEFAULT_KEY_ALIAS = "server"; @@ -72,7 +72,7 @@ public class PemSslCredentials extends AbstractSslCredentials { try (InputStream inStream = ResourceUtils.getInputStream(this, this.certFile)) { try (PEMParser pemParser = new PEMParser(new InputStreamReader(inStream))) { Object object; - while((object = pemParser.readObject()) != null) { + while ((object = pemParser.readObject()) != null) { if (object instanceof X509CertificateHolder) { X509Certificate x509Cert = certConverter.getCertificate((X509CertificateHolder) object); certificates.add(x509Cert); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 777a77d054..9803670d4b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.dao.service.Validator; @@ -41,7 +42,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; import static org.thingsboard.server.dao.attributes.AttributeUtils.validate; @@ -101,26 +101,29 @@ public class BaseAttributesService implements AttributesService { } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { validate(entityId, scope); AttributeUtils.validate(attribute, valueNoXssValidation); - return doSave(tenantId, entityId, scope, attribute); + return doSave(tenantId, entityId, scope, List.of(attribute)); } @Override - public ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { validate(entityId, scope); AttributeUtils.validate(attributes, valueNoXssValidation); - List> saveFutures = attributes.stream().map(attribute -> doSave(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); - return Futures.allAsList(saveFutures); + return doSave(tenantId, entityId, scope, attributes); } - private ListenableFuture doSave(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { - ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); - return Futures.transform(future, version -> { - edqsService.onUpdate(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attribute, version)); - return version; - }, MoreExecutors.directExecutor()); + private ListenableFuture doSave(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { + List> futures = new ArrayList<>(attributes.size()); + for (AttributeKvEntry attribute : attributes) { + ListenableFuture future = Futures.transform(attributesDao.save(tenantId, entityId, scope, attribute), version -> { + edqsService.onUpdate(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attribute, version)); + return version; + }, MoreExecutors.directExecutor()); + futures.add(future); + } + return Futures.transform(Futures.allAsList(futures), AttributesSaveResult::of, MoreExecutors.directExecutor()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 559828911f..d99413f13a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.msg.edqs.EdqsService; @@ -56,7 +57,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; import static org.thingsboard.server.dao.attributes.AttributeUtils.validate; @@ -150,7 +150,7 @@ public class CachedAttributesService implements AttributesService { List cachedAttributes = wrappedCachedAttributes.values().stream() .map(TbCacheValueWrapper::get) .filter(Objects::nonNull) - .collect(Collectors.toList()); + .toList(); if (wrappedCachedAttributes.size() == attributeKeys.size()) { log.trace("[{}][{}] Found all attributes from cache: {}", entityId, scope, attributeKeys); return Futures.immediateFuture(cachedAttributes); @@ -159,8 +159,6 @@ public class CachedAttributesService implements AttributesService { Set notFoundAttributeKeys = new HashSet<>(attributeKeys); notFoundAttributeKeys.removeAll(wrappedCachedAttributes.keySet()); - List notFoundKeys = notFoundAttributeKeys.stream().map(k -> new AttributeCacheKey(scope, entityId, k)).collect(Collectors.toList()); - // DB call should run in DB executor, not in cache-related executor return jpaExecutorService.submit(() -> { log.trace("[{}][{}] Lookup attributes from db: {}", entityId, scope, notFoundAttributeKeys); @@ -222,33 +220,31 @@ public class CachedAttributesService implements AttributesService { } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { validate(entityId, scope); AttributeUtils.validate(attribute, valueNoXssValidation); - return doSave(tenantId, entityId, scope, attribute); + return doSave(tenantId, entityId, scope, List.of(attribute)); } @Override - public ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { validate(entityId, scope); AttributeUtils.validate(attributes, valueNoXssValidation); + return doSave(tenantId, entityId, scope, attributes); + } + private ListenableFuture doSave(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { List> futures = new ArrayList<>(attributes.size()); for (var attribute : attributes) { - futures.add(doSave(tenantId, entityId, scope, attribute)); + ListenableFuture future = Futures.transform(attributesDao.save(tenantId, entityId, scope, attribute), version -> { + BaseAttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(((BaseAttributeKvEntry) attribute).getKv(), attribute.getLastUpdateTs(), version); + put(entityId, scope, attributeKvEntry); + edqsService.onUpdate(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attributeKvEntry, version)); + return version; + }, cacheExecutor); + futures.add(future); } - - return Futures.allAsList(futures); - } - - private ListenableFuture doSave(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { - ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); - return Futures.transform(future, version -> { - BaseAttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(((BaseAttributeKvEntry) attribute).getKv(), attribute.getLastUpdateTs(), version); - put(entityId, scope, attributeKvEntry); - edqsService.onUpdate(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attributeKvEntry, version)); - return version; - }, cacheExecutor); + return Futures.transform(Futures.allAsList(futures), AttributesSaveResult::of, MoreExecutors.directExecutor()); } private void put(EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { @@ -270,7 +266,7 @@ public class CachedAttributesService implements AttributesService { edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); } return key; - }, cacheExecutor)).collect(Collectors.toList())); + }, cacheExecutor)).toList()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java index 48a1eb6d5a..801c436d2f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java @@ -16,10 +16,8 @@ package org.thingsboard.server.dao.sql.attributes; import org.springframework.stereotype.Repository; -import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.AbstractVersionedInsertRepository; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -27,8 +25,6 @@ import java.sql.Types; import java.util.List; @Repository -@Transactional -@SqlDao public class AttributeKvInsertRepository extends AbstractVersionedInsertRepository { private static final String BATCH_UPDATE = "UPDATE attribute_kv SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ?, version = nextval('attribute_kv_version_seq') " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index e59ff3f4e6..bbce4e2721 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -79,7 +79,6 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); long version = row.get("version") != null ? (long) row.get("version") : 0; String debugSettings = (String) row.get("debug_settings"); - Object externalIdObj = row.get("external_id"); CalculatedField calculatedField = new CalculatedField(); calculatedField.setId(new CalculatedFieldId(id)); @@ -89,7 +88,12 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF calculatedField.setType(type); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); + try { + calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); + } catch (Exception e) { + log.error("Invalid configuration for CalculatedField [{}]. Skipping.", id, e); + return null; + } calculatedField.setVersion(version); calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); @@ -118,7 +122,6 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); UUID entityId = (UUID) row.get("entity_id"); UUID calculatedFieldId = (UUID) row.get("calculated_field_id"); - JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); calculatedFieldLink.setId(new CalculatedFieldLinkId(id)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java index 53a824f026..c14c069f23 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java @@ -29,7 +29,6 @@ import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; -import org.thingsboard.server.dao.util.SqlDao; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; @@ -38,7 +37,6 @@ import java.util.concurrent.locks.ReentrantLock; @Component @Slf4j -@SqlDao @RequiredArgsConstructor public class JpaKeyDictionaryDao extends JpaAbstractDaoListeningExecutorService implements KeyDictionaryDao { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java index b40fa520f2..c9d16f00ec 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java @@ -18,11 +18,9 @@ package org.thingsboard.server.dao.sqlts.insert.latest.sql; import jakarta.annotation.PostConstruct; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Repository; -import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.AbstractVersionedInsertRepository; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; -import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; import java.sql.PreparedStatement; @@ -32,8 +30,6 @@ import java.util.List; @SqlTsLatestAnyDao @Repository -@Transactional -@SqlDao public class SqlLatestInsertTsRepository extends AbstractVersionedInsertRepository implements InsertLatestTsRepository { @Value("${sql.ts_latest.update_by_latest_ts:true}") diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/KvUtils.java b/dao/src/main/java/org/thingsboard/server/dao/util/KvUtils.java index eeb88959fa..8b95ddcb57 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/KvUtils.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/KvUtils.java @@ -33,9 +33,8 @@ public class KvUtils { static { validatedKeys = Caffeine.newBuilder() - .weakKeys() .expireAfterAccess(24, TimeUnit.HOURS) - .maximumSize(100000).build(); + .maximumSize(50000).build(); } public static void validate(List tsKvEntries, boolean valueNoXssValidation) { @@ -57,11 +56,13 @@ public class KvUtils { throw new DataValidationException("Validation error: key length must be equal or less than 255"); } - if (validatedKeys.getIfPresent(key) == null) { - if (!NoXssValidator.isValid(key)) { - throw new DataValidationException("Validation error: key is malformed"); - } - validatedKeys.put(key, Boolean.TRUE); + Boolean isValid = validatedKeys.asMap().get(key); + if (isValid == null) { + isValid = NoXssValidator.isValid(key); + validatedKeys.put(key, isValid); + } + if (!isValid) { + throw new DataValidationException("Validation error: key is malformed"); } if (valueNoXssValidation) { diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java index 6c7444c92a..f4c256a8cf 100644 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java @@ -70,7 +70,45 @@ public class RepositoryUtilsTest { Arguments.of("loranet 123", getNameFilter(StringOperation.IN, "loranet 123, loranet 124"), true), Arguments.of("loranet 123", getNameFilter(StringOperation.IN, "loranet 125, loranet 126"), false), Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 125, loranet 126"), true), - Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 123, loranet 126"), false) + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 123, loranet 126"), false), + + // Basic CONTAINS + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%loranet"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet%"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%ranet%"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%123"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%loranx%"), false), + + // Basic STARTS_WITH + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "loranet%"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "lora%"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "lorax%"), false), + + // Basic ENDS_WITH + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "%123"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "%23"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "%124"), false), + + // CONTAINS with _ + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet_123"), true), // '_' = ' ' + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet_12_"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loran_t%"), true), + + // STARTS_WITH with _ + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "loranet_"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "lora__t%"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "lor_net%"), true), + + // ENDS_WITH with _ + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "_23"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "_2_"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "_3"), true), + + // Mixed patterns + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "lora__t 1%"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "lora%net%3"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%o_anet%2_3"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "lora___ ___"), true) ); } diff --git a/pom.xml b/pom.xml index c4c71cfa63..64cdd63eaf 100755 --- a/pom.xml +++ b/pom.xml @@ -42,19 +42,19 @@ 4.0.2 2.4.0-b180830.0359 4.0.5 - 10.1.40 + 10.1.42 2.5.2 3.2.12 3.2.12 3.2.12 - 6.1.15 + 6.1.21 6.2.11 - 6.3.8 + 6.3.9 5.1.5 0.12.5 - 2.0.13 - 2.23.1 - 1.5.5 + 2.0.17 + 2.24.3 + 1.5.6 0.10 4.17.0 4.2.25 @@ -83,7 +83,7 @@ 2.3.32 2.0.1 5.6.0 - 3.9.2 + 3.9.3 3.25.5 1.63.0 1.2.6 @@ -103,7 +103,7 @@ 1.19.0 1.78.1 2.0.1 - 42.7.5 + 42.7.7 org/thingsboard/server/gen/**/*, org/thingsboard/server/extensions/core/plugin/telemetry/gen/**/* @@ -113,7 +113,7 @@ - 3.7.2 + 3.9.1 8.10.1 3.5.3 2.2 @@ -158,7 +158,7 @@ 2.12.0 3.0.2 - 6.9.0.202403050737-r + 6.10.1.202505221210-r 0.4.8 1.0.0 @@ -1178,7 +1178,7 @@ org.springframework.boot - spring-boot-starter + spring-boot-starter ${spring-boot.version} diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 3816b52963..67e11430bd 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -102,6 +102,7 @@ import org.thingsboard.server.common.data.id.AlarmCommentId; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceId; @@ -4073,6 +4074,19 @@ public class RestClient implements Closeable { return restTemplate.postForEntity(baseURL + "/api/calculatedField", calculatedField, CalculatedField.class).getBody(); } + public Optional getCalculatedFieldById(CalculatedFieldId calculatedFieldId) { + try { + ResponseEntity calculatedField = restTemplate.getForEntity(baseURL + "/api/calculatedField/{calculatedFieldId}", CalculatedField.class, calculatedFieldId.getId()); + return Optional.ofNullable(calculatedField.getBody()); + } catch (HttpClientErrorException exception) { + if (exception.getStatusCode() == HttpStatus.NOT_FOUND) { + return Optional.empty(); + } else { + throw exception; + } + } + } + public PageData getCalculatedFieldsByEntityId(EntityId entityId, PageLink pageLink) { Map params = new HashMap<>(); addPageLinkToParam(params, pageLink); @@ -4084,6 +4098,36 @@ public class RestClient implements Closeable { } + public void deleteCalculatedField(CalculatedFieldId calculatedFieldId) { + restTemplate.delete(baseURL + "/api/calculatedField/{calculatedFieldId}", calculatedFieldId.getId()); + } + + public Optional getLatestCalculatedFieldDebugEvent(CalculatedFieldId calculatedFieldId) { + try { + ResponseEntity jsonNode = restTemplate.getForEntity(baseURL + "/api/calculatedField/{calculatedFieldId}/debug", JsonNode.class, calculatedFieldId.getId()); + return Optional.ofNullable(jsonNode.getBody()); + } catch (HttpClientErrorException exception) { + if (exception.getStatusCode() == HttpStatus.NOT_FOUND) { + return Optional.empty(); + } else { + throw exception; + } + } + } + + public Optional testCalculatedFieldScript(JsonNode inputParams) { + try { + ResponseEntity jsonNode = restTemplate.postForEntity(baseURL + "/api/calculatedField/testScript", inputParams, JsonNode.class); + return Optional.ofNullable(jsonNode.getBody()); + } catch (HttpClientErrorException exception) { + if (exception.getStatusCode() == HttpStatus.NOT_FOUND) { + return Optional.empty(); + } else { + throw exception; + } + } + } + private String getTimeUrlParams(TimePageLink pageLink) { String urlParams = getUrlParams(pageLink); if (pageLink.getStartTime() != null) { diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java index c3095836bd..19c1956463 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java @@ -57,6 +57,7 @@ public class AttributesSaveRequest implements CalculatedFieldSystemAwareRequest public static final Strategy PROCESS_ALL = new Strategy(true, true, true); public static final Strategy WS_ONLY = new Strategy(false, true, false); public static final Strategy SKIP_ALL = new Strategy(false, false, false); + public static final Strategy CF_ONLY = new Strategy(false, false, true); } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index c402a0c984..3bacf8d8c3 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -56,6 +56,7 @@ public class TimeseriesSaveRequest implements CalculatedFieldSystemAwareRequest public static final Strategy WS_ONLY = new Strategy(false, false, true, false); public static final Strategy LATEST_AND_WS = new Strategy(false, true, true, false); public static final Strategy SKIP_ALL = new Strategy(false, false, false, false); + public static final Strategy CF_ONLY = new Strategy(false, false, false, true); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java index b2cce52c87..4e319500d1 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java @@ -92,13 +92,14 @@ public class TbCalculatedFieldsNode implements TbNode { .customerId(msg.getCustomerId()) .entityId(msg.getOriginator()) .entries(tsKvEntryList) + .strategy(TimeseriesSaveRequest.Strategy.CF_ONLY) .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) .tbMsgId(msg.getId()) .tbMsgType(msg.getInternalType()) .callback(new TelemetryNodeCallback(ctx, msg)) .build(); - ctx.getCalculatedFieldQueueService().pushRequestToQueue(timeseriesSaveRequest, timeseriesSaveRequest.getCallback()); + ctx.getTelemetryService().saveTimeseries(timeseriesSaveRequest); } private void processPostAttributesRequest(TbContext ctx, TbMsg msg) { @@ -114,12 +115,13 @@ public class TbCalculatedFieldsNode implements TbNode { .entityId(msg.getOriginator()) .scope(AttributeScope.valueOf(msg.getMetaData().getValue(SCOPE))) .entries(newAttributes) + .strategy(AttributesSaveRequest.Strategy.CF_ONLY) .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) .tbMsgId(msg.getId()) .tbMsgType(msg.getInternalType()) .callback(new TelemetryNodeCallback(ctx, msg)) .build(); - ctx.getCalculatedFieldQueueService().pushRequestToQueue(attributesSaveRequest, attributesSaveRequest.getCallback()); + ctx.getTelemetryService().saveAttributes(attributesSaveRequest); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 60f33ff005..7b69d26a60 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -190,9 +190,9 @@
- -
- calculated-fields.use-message-timestamp + +
+ calculated-fields.use-latest-timestamp
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss index efcd62efd4..e192e3ccc0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss @@ -45,7 +45,7 @@ &-key { color: #c24c1a; } - &-time-window, &-values, &-func, &-value, &-ts, &-msgTs { + &-time-window, &-values, &-func, &-value, &-ts, &-latestTs { color: #7214D0; } &-start-ts, &-end-ts { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 4aa4eca425..975744b2c6 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -77,7 +77,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent
- + rule-node-config.credentials diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/gauge/digital-simple-gauge-basic-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/gauge/digital-simple-gauge-basic-config.component.ts index 67497b0b6e..7d348662ae 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/gauge/digital-simple-gauge-basic-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/gauge/digital-simple-gauge-basic-config.component.ts @@ -30,7 +30,7 @@ import { getTimewindowConfig, setTimewindowConfig } from '@home/components/widget/config/timewindow-config-panel.component'; -import { formatValue, isUndefined } from '@core/utils'; +import { formatValue, isDefined, isUndefined } from '@core/utils'; import { Component } from '@angular/core'; import { convertLevelColorsSettingsToColorProcessor, @@ -115,7 +115,7 @@ export class DigitalSimpleGaugeBasicConfigComponent extends BasicWidgetConfigCom minMaxColor: [settings.minMaxFont?.color, []], showValue: [settings.showValue, []], - decimals: [configData.config.decimals, []], + decimals: [isDefined(configData.config.decimals) ? configData.config.decimals : settings.decimals, []], units: [configData.config.units, []], valueFont: [settings.valueFont, []], valueColor: [settings.valueFont?.color, []], @@ -157,6 +157,9 @@ export class DigitalSimpleGaugeBasicConfigComponent extends BasicWidgetConfigCom this.widgetConfig.config.settings.showValue = config.showValue; this.widgetConfig.config.units = config.units; this.widgetConfig.config.decimals = config.decimals; + if (isDefined(this.widgetConfig.config.settings.decimals)) { + delete this.widgetConfig.config.settings.decimals; + } this.widgetConfig.config.settings.valueFont = config.valueFont; this.widgetConfig.config.settings.valueFont.color = config.valueColor; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.ts index 8c1fc3ab2a..f28b5e8f92 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/knob.component.ts @@ -57,6 +57,7 @@ export class KnobComponent extends BasicActionWidgetComponent implements OnInit, maxValue: number; newValue = 0; + private decimals: number; private startDeg = -1; private currentDeg = 0; private rotation = 0; @@ -143,9 +144,10 @@ export class KnobComponent extends BasicActionWidgetComponent implements OnInit, actionLabel: this.ctx.translate.instant('widgets.slider.on-value-change')}; this.valueSetter = this.createValueSetter(valueChangeSettings); + this.decimals = isDefined(this.ctx.decimals) ? this.ctx.decimals : 0; this.valueFormat = ValueFormatProcessor.fromSettings(this.ctx.$injector, { units: this.ctx.units, - decimals: this.ctx.decimals, + decimals: this.decimals, showZeroDecimals: true }); @@ -299,7 +301,7 @@ export class KnobComponent extends BasicActionWidgetComponent implements OnInit, } private turn(ratio: number) { - this.newValue = Number((this.minValue + (this.maxValue - this.minValue) * ratio).toFixed(this.ctx.decimals)); + this.newValue = Number((this.minValue + (this.maxValue - this.minValue) * ratio).toFixed(this.decimals)); if (this.canvasBar.value !== this.newValue) { this.canvasBar.value = this.newValue; } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/control/knob-control-widget-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/control/knob-control-widget-settings.component.ts index e5c57474c2..26a5d6cd4e 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/control/knob-control-widget-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/control/knob-control-widget-settings.component.ts @@ -57,7 +57,7 @@ export class KnobControlWidgetSettingsComponent extends WidgetSettingsComponent protected prepareInputSettings(settings: WidgetSettings): WidgetSettings { const knobSettings = prepareKnobSettings(deepClone(settings) as any) as WidgetSettings; - knobSettings.valueDecimals = this.widgetConfig?.config?.decimals ?? 2; + knobSettings.valueDecimals = this.widgetConfig?.config?.decimals; knobSettings.valueUnits = deepClone(this.widgetConfig?.config?.units); return super.prepareInputSettings(knobSettings); } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/gauge/digital-gauge-widget-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/gauge/digital-gauge-widget-settings.component.ts index 7276c0e27d..1f3ae36089 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/gauge/digital-gauge-widget-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/gauge/digital-gauge-widget-settings.component.ts @@ -37,7 +37,7 @@ import { digitalGaugeLayoutTranslations, DigitalGaugeType } from '@home/components/widget/lib/digital-gauge.models'; -import { formatValue } from '@core/utils'; +import { formatValue, isDefined } from '@core/utils'; import { ColorSettings, ColorType, @@ -247,6 +247,10 @@ export class DigitalGaugeWidgetSettingsComponent extends WidgetSettingsComponent settings.titleFont.color = this.digitalGaugeWidgetSettingsForm.get('titleColor').value; settings.labelFont.color = this.digitalGaugeWidgetSettingsForm.get('labelColor').value; + if (isDefined(settings.decimals)) { + delete settings.decimals; + } + return settings; } diff --git a/ui-ngx/src/app/shared/import-export/import-export.models.ts b/ui-ngx/src/app/shared/import-export/import-export.models.ts index 2f20a421c3..60df84c737 100644 --- a/ui-ngx/src/app/shared/import-export/import-export.models.ts +++ b/ui-ngx/src/app/shared/import-export/import-export.models.ts @@ -160,7 +160,7 @@ export const ZIP_TYPE: FileType = { }; export const CSV_TYPE: FileType = { - mimeType: 'attachament/csv', + mimeType: 'text/csv', extension: 'csv' }; diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 6fade1d193..76b775b2fd 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -526,10 +526,10 @@ export const getCalculatedFieldArgumentsEditorCompleter = (argumentsObj: Record< description: 'Calculated field context arguments.', children: {} }, - msgTs: { + latestTs: { meta: 'constant', type: 'number', - description: 'Timestamp (ms) of the telemetry message that triggered the calculated field execution.' + description: 'Latest timestamp (ms) of the arguments telemetry.' } } } @@ -582,8 +582,8 @@ const calculatedFieldArgumentsContextValueHighlightRules: AceHighlightRules = { next: 'calculatedFieldCtxArgs' }, { - token: 'tb.calculated-field-msgTs', - regex: /msgTs/, + token: 'tb.calculated-field-latestTs', + regex: /latestTs/, next: 'no_regex' }, endGroupHighlightRule diff --git a/ui-ngx/src/app/shared/models/time/time.models.ts b/ui-ngx/src/app/shared/models/time/time.models.ts index acb62d9d2a..c204cb6867 100644 --- a/ui-ngx/src/app/shared/models/time/time.models.ts +++ b/ui-ngx/src/app/shared/models/time/time.models.ts @@ -431,7 +431,7 @@ export const initModelFromDefaultTimewindow = (value: Timewindow, quickIntervalO if (isDefined(value.history.quickInterval)) { model.history.quickInterval = value.history.quickInterval; } - if (isDefined(value.history.fixedTimewindow)) { + if (isDefinedAndNotNull(value.history.fixedTimewindow)) { if (isDefined(value.history.fixedTimewindow.startTimeMs)) { model.history.fixedTimewindow.startTimeMs = value.history.fixedTimewindow.startTimeMs; } diff --git a/ui-ngx/src/app/shared/models/widget/rpc/knob.component.models.ts b/ui-ngx/src/app/shared/models/widget/rpc/knob.component.models.ts index 47b73f5a08..3ac15bf70c 100644 --- a/ui-ngx/src/app/shared/models/widget/rpc/knob.component.models.ts +++ b/ui-ngx/src/app/shared/models/widget/rpc/knob.component.models.ts @@ -45,7 +45,7 @@ export const knobWidgetDefaultSettings: KnobSettings = { defaultValue: 50, executeRpc: { method: 'getValue', - requestTimeout: 500, + requestTimeout: 5000, requestPersistent: false, persistentPollingInterval: 5000 }, @@ -70,7 +70,7 @@ export const knobWidgetDefaultSettings: KnobSettings = { action: SetValueAction.EXECUTE_RPC, executeRpc: { method: 'setValue', - requestTimeout: 500, + requestTimeout: 5000, requestPersistent: false, persistentPollingInterval: 5000 }, diff --git a/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md b/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md index 4c54b01499..1d45dea3c8 100644 --- a/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md +++ b/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md @@ -1,7 +1,7 @@ ## Calculated Field TBEL Script Function The **calculate()** function is a user-defined script that enables custom calculations using [TBEL](${siteBaseUrl}/docs${docPlatformPrefix}/user-guide/tbel/) on telemetry and attribute data. -It receives arguments configured in the calculated field setup, along with an additional `ctx` object that stores `msgTs` and provides access to all arguments. +It receives arguments configured in the calculated field setup, along with an additional `ctx` object that stores `latestTs` and provides access to all arguments. ### Function Signature @@ -216,14 +216,14 @@ The return format depends on the output type configured in the calculated field ### Message timestamp -The `ctx` object also includes property `msgTs`, which represents the timestamp of the incoming telemetry message that triggered the calculated field execution in milliseconds. +The `ctx` object also includes property `latestTs`, which represents the latest timestamp of the arguments telemetry in milliseconds. -You can use `ctx.msgTs` to set the timestamp of the resulting output explicitly when returning a time series object. +You can use `ctx.latestTs` to set the timestamp of the resulting output explicitly when returning a time series object. ```javascript var temperatureC = (temperatureF - 32) / 1.8; return { - ts: ctx.msgTs, + ts: ctx.latestTs, values: { "temperatureC": toFixed(temperatureC, 2) } diff --git a/ui-ngx/src/assets/help/en_US/rulenode/switch_node_script_fn.md b/ui-ngx/src/assets/help/en_US/rulenode/switch_node_script_fn.md index c4092455fa..49f0dd0a2b 100644 --- a/ui-ngx/src/assets/help/en_US/rulenode/switch_node_script_fn.md +++ b/ui-ngx/src/assets/help/en_US/rulenode/switch_node_script_fn.md @@ -91,11 +91,5 @@ return []; {:copy-code} ``` -
- -You can see real life example, how to use this node in this tutorial: - -- [Data function based on telemetry from 2 devices{:target="_blank"}](${siteBaseUrl}/docs${docPlatformPrefix}/user-guide/rule-engine-2-0/tutorials/function-based-on-telemetry-from-two-devices#delta-temperature-rule-chain) -

diff --git a/ui-ngx/src/assets/help/en_US/rulenode/tbel/switch_node_script_fn.md b/ui-ngx/src/assets/help/en_US/rulenode/tbel/switch_node_script_fn.md index 2461f4cace..77da936c73 100644 --- a/ui-ngx/src/assets/help/en_US/rulenode/tbel/switch_node_script_fn.md +++ b/ui-ngx/src/assets/help/en_US/rulenode/tbel/switch_node_script_fn.md @@ -91,11 +91,5 @@ return []; {:copy-code} ``` -
- -You can see real life example, how to use this node in this tutorial: - -- [Data function based on telemetry from 2 devices{:target="_blank"}](${siteBaseUrl}/docs${docPlatformPrefix}/user-guide/rule-engine-2-0/tutorials/function-based-on-telemetry-from-two-devices#delta-temperature-rule-chain) -

diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index f5ed1d6d8e..a98b2aecf9 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1069,7 +1069,7 @@ "delete-multiple-title": "Are you sure you want to delete { count, plural, =1 {1 calculated field} other {# calculated fields} }?", "delete-multiple-text": "Be careful, after the confirmation all selected calculated fields will be removed and all related data will become unrecoverable.", "test-with-this-message": "Test with this message", - "use-message-timestamp": "Use message timestamp", + "use-latest-timestamp": "Use latest timestamp", "hint": { "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with time series rolling type.", "arguments-empty": "Arguments should not be empty.", @@ -1086,7 +1086,7 @@ "decimals-range": "Decimals by default should be a number between 0 and 15.", "expression": "Default expression demonstrates how to transform a temperature from Fahrenheit to Celsius.", "arguments-entity-not-found": "Argument target entity not found.", - "use-message-timestamp": "If enabled, the calculated value will be persisted using the timestamp of the telemetry that triggered the calculation, instead of the server time." + "use-latest-timestamp": "If enabled, the calculated value will be persisted using the most recent timestamp from the arguments telemetry, instead of the server time." } }, "confirm-on-exit": {