From e0fd611c768d98ef6475f9872aaa08bb533bad82 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 13 Dec 2024 14:35:02 +0100 Subject: [PATCH 01/40] used TbMsgProto instead of ByteString --- .../actors/ruleChain/DefaultTbContext.java | 5 +- .../RuleChainActorMessageProcessor.java | 2 +- .../RuleNodeActorMessageProcessor.java | 2 +- .../device/DeviceProvisionServiceImpl.java | 3 +- ...riginatorIdTbRuleEngineSubmitStrategy.java | 7 +- ...TbRuleEngineProcessingStrategyFactory.java | 6 +- .../TbRuleEngineQueueConsumerManager.java | 14 +++- .../rpc/DefaultTbRuleEngineRpcService.java | 2 +- .../DefaultRuleEngineCallService.java | 2 +- .../actors/rule/DefaultTbContextTest.java | 12 +-- .../controller/TenantControllerTest.java | 3 +- .../TbRuleEngineQueueConsumerManagerTest.java | 2 +- .../ruleengine/TbRuleEngineStrategyTest.java | 2 +- .../DefaultTbRuleEngineRpcServiceTest.java | 2 +- .../DefaultRuleEngineCallServiceTest.java | 2 +- .../thingsboard/server/common/msg/TbMsg.java | 82 ++++++++++--------- common/proto/src/main/proto/queue.proto | 8 +- .../common/TbRuleEngineProducerService.java | 2 +- 18 files changed, 90 insertions(+), 68 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 0070eba4a1..6ebcc1843e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -230,7 +230,8 @@ public class DefaultTbContext implements TbContext { TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + .setTbMsgProto(TbMsg.toProto(tbMsg)) + .build(); mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, callback); } @@ -309,7 +310,7 @@ public class DefaultTbContext implements TbContext { TransportProtos.ToRuleEngineMsg.Builder msg = TransportProtos.ToRuleEngineMsg.newBuilder() .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(tbMsg)) + .setTbMsgProto(TbMsg.toProto(tbMsg)) .addAllRelationTypes(relationTypes); if (failureMessage != null) { msg.setFailureMessage(failureMessage); diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 460da228c3..917066cd2d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -370,7 +370,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor(tbMsg.getId(), msg), callback); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java index 1ee6df8361..3c11095894 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java @@ -34,7 +34,12 @@ public class SequentialByOriginatorIdTbRuleEngineSubmitStrategy extends Sequenti @Override protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { try { - MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(msg.getTbMsg()); + MsgProtos.TbMsgProto proto; + if (msg.getTbMsg().isEmpty()) { + proto = msg.getTbMsgProto(); + } else { + proto = MsgProtos.TbMsgProto.parseFrom(msg.getTbMsg()); + } return EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); } catch (InvalidProtocolBufferException e) { log.warn("[{}] Failed to parse TbMsg: {}", queueName, msg); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java index 4712497524..597bcef098 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -125,7 +125,7 @@ public class TbRuleEngineProcessingStrategyFactory { } log.debug("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); if (log.isTraceEnabled()) { - toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(result.getQueueName(), msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); + toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromProto(result.getQueueName(), msg.getValue().getTbMsgProto(), msg.getValue().getTbMsg(), TbMsgCallback.EMPTY))); } if (pauseBetweenRetries > 0) { try { @@ -164,10 +164,10 @@ public class TbRuleEngineProcessingStrategyFactory { log.debug("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); } if (log.isTraceEnabled()) { - result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(result.getQueueName(), msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); + result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromProto(result.getQueueName(), msg.getValue().getTbMsgProto(), msg.getValue().getTbMsg(), TbMsgCallback.EMPTY))); } if (log.isTraceEnabled()) { - result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, TbMsg.fromBytes(result.getQueueName(), msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); + result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, TbMsg.fromProto(result.getQueueName(), msg.getValue().getTbMsgProto(), msg.getValue().getTbMsg(), TbMsgCallback.EMPTY))); } return new TbRuleEngineProcessingDecision(true, null); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index c2823d3c00..231d7b205d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -170,7 +170,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager relationTypes; @@ -199,7 +199,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager> pending : map.entrySet()) { ToRuleEngineMsg tmp = pending.getValue().getValue(); - TbMsg tmpMsg = TbMsg.fromBytes(config.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY); + TbMsg tmpMsg = TbMsg.fromProto(config.getName(), tmp.getTbMsgProto(), tmp.getTbMsg(), TbMsgCallback.EMPTY); RuleNodeInfo ruleNodeInfo = ctx.getLastVisitedRuleNode(pending.getKey()); if (printAll) { log.trace("[{}][{}] {} to process message: {}, Last Rule Node: {}", queueKey, TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); @@ -228,7 +228,13 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager msg : msgs) { try { - MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); + MsgProtos.TbMsgProto tbMsgProto; + if (msg.getValue().getTbMsg().isEmpty()) { + tbMsgProto = msg.getValue().getTbMsgProto(); + } else { + tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg()); + } + EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); TopicPartitionInfo tpi = ctx.getPartitionService().resolve(ServiceType.TB_RULE_ENGINE, config.getName(), TenantId.SYS_TENANT_ID, originator); diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java index debc1cb987..80b3bd7a37 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java @@ -132,7 +132,7 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi TransportProtos.RestApiCallResponseMsgProto msg = TransportProtos.RestApiCallResponseMsgProto.newBuilder() .setRequestIdMSB(requestId.getMostSignificantBits()) .setRequestIdLSB(requestId.getLeastSignificantBits()) - .setResponse(TbMsg.toByteString(tbMsg)) + .setResponseProto(TbMsg.toProto(tbMsg)) .build(); clusterService.pushNotificationToCore(serviceId, msg, null); } diff --git a/application/src/main/java/org/thingsboard/server/service/ruleengine/DefaultRuleEngineCallService.java b/application/src/main/java/org/thingsboard/server/service/ruleengine/DefaultRuleEngineCallService.java index 6f4d18b94e..e435c8ad36 100644 --- a/application/src/main/java/org/thingsboard/server/service/ruleengine/DefaultRuleEngineCallService.java +++ b/application/src/main/java/org/thingsboard/server/service/ruleengine/DefaultRuleEngineCallService.java @@ -73,7 +73,7 @@ public class DefaultRuleEngineCallService implements RuleEngineCallService { UUID requestId = new UUID(restApiCallResponseMsg.getRequestIdMSB(), restApiCallResponseMsg.getRequestIdLSB()); Consumer consumer = requests.remove(requestId); if (consumer != null) { - consumer.accept(TbMsg.fromBytes(null, restApiCallResponseMsg.getResponse().toByteArray(), TbMsgCallback.EMPTY)); + consumer.accept(TbMsg.fromProto(null, restApiCallResponseMsg.getResponseProto(), restApiCallResponseMsg.getResponse(), TbMsgCallback.EMPTY)); } else { log.trace("[{}] Unknown or stale rest api call response received", requestId); } diff --git a/application/src/test/java/org/thingsboard/server/actors/rule/DefaultTbContextTest.java b/application/src/test/java/org/thingsboard/server/actors/rule/DefaultTbContextTest.java index 21a4a45c6f..4370b98b2d 100644 --- a/application/src/test/java/org/thingsboard/server/actors/rule/DefaultTbContextTest.java +++ b/application/src/test/java/org/thingsboard/server/actors/rule/DefaultTbContextTest.java @@ -644,11 +644,11 @@ class DefaultTbContextTest { ToRuleEngineMsg actualToRuleEngineMsg = toRuleEngineMsgCaptor.getValue(); assertThat(actualToRuleEngineMsg).usingRecursiveComparison() - .ignoringFields("tbMsg_") + .ignoringFields("tbMsgProto_") .isEqualTo(ToRuleEngineMsg.newBuilder() .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) .setTenantIdLSB(TENANT_ID.getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(expectedTbMsg)) + .setTbMsgProto(TbMsg.toProto(expectedTbMsg)) .addAllRelationTypes(List.of(connectionType)).build()); var simpleTbQueueCallback = simpleTbQueueCallbackCaptor.getValue(); @@ -701,11 +701,11 @@ class DefaultTbContextTest { ToRuleEngineMsg actualToRuleEngineMsg = toRuleEngineMsgCaptor.getValue(); assertThat(actualToRuleEngineMsg).usingRecursiveComparison() - .ignoringFields("tbMsg_") + .ignoringFields("tbMsgProto_") .isEqualTo(ToRuleEngineMsg.newBuilder() .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) .setTenantIdLSB(TENANT_ID.getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(expectedTbMsg)) + .setTbMsgProto(TbMsg.toProto(expectedTbMsg)) .build()); var simpleTbQueueCallback = simpleTbQueueCallbackCaptor.getValue(); @@ -883,11 +883,11 @@ class DefaultTbContextTest { ToRuleEngineMsg actualToRuleEngineMsg = toRuleEngineMsgCaptor.getValue(); assertThat(actualToRuleEngineMsg).usingRecursiveComparison() - .ignoringFields("tbMsg_") + .ignoringFields("tbMsgProto_.id_") .isEqualTo(ToRuleEngineMsg.newBuilder() .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) .setTenantIdLSB(TENANT_ID.getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(expectedTbMsg)) + .setTbMsgProto(TbMsg.toProto(expectedTbMsg)) .setFailureMessage(EXCEPTION_MSG) .addAllRelationTypes(List.of(TbNodeConnectionType.FAILURE)).build()); diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java index c8cc72c5b5..dbf40c0e30 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java @@ -747,7 +747,8 @@ public class TenantControllerTest extends AbstractControllerTest { TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + .setTbMsgProto(TbMsg.toProto(tbMsg)) + .build(); tbClusterService.pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, null); return tbMsg; } diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java index 66e3de13d1..bff5d165f4 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -738,7 +738,7 @@ public class TbRuleEngineQueueConsumerManagerTest { .setTenantIdMSB(tenantId.getMostSignificantBits()) .setTenantIdLSB(tenantId.getLeastSignificantBits()) .addRelationTypes("Success") - .setTbMsg(TbMsg.toByteString(tbMsg)) + .setTbMsgProto(TbMsg.toProto(tbMsg)) .build()); } diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineStrategyTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineStrategyTest.java index 098c622e31..435304fdef 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineStrategyTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineStrategyTest.java @@ -263,7 +263,7 @@ public class TbRuleEngineStrategyTest { .setTenantIdMSB(tenantId.getMostSignificantBits()) .setTenantIdLSB(tenantId.getLeastSignificantBits()) .addRelationTypes("Success") - .setTbMsg(TbMsg.toByteString(tbMsg)) + .setTbMsgProto(TbMsg.toProto(tbMsg)) .build()); } diff --git a/application/src/test/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcServiceTest.java b/application/src/test/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcServiceTest.java index 2cab5f991a..ea0b3b91f5 100644 --- a/application/src/test/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcServiceTest.java @@ -50,7 +50,7 @@ class DefaultTbRuleEngineRpcServiceTest { var restApiCallResponseMsgProto = TransportProtos.RestApiCallResponseMsgProto.newBuilder() .setRequestIdMSB(requestId.getMostSignificantBits()) .setRequestIdLSB(requestId.getLeastSignificantBits()) - .setResponse(TbMsg.toByteString(msg)) + .setResponseProto(TbMsg.toProto(msg)) .build(); // WHEN diff --git a/application/src/test/java/org/thingsboard/server/service/ruleengine/DefaultRuleEngineCallServiceTest.java b/application/src/test/java/org/thingsboard/server/service/ruleengine/DefaultRuleEngineCallServiceTest.java index c49149bd07..a4ee9cda71 100644 --- a/application/src/test/java/org/thingsboard/server/service/ruleengine/DefaultRuleEngineCallServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/ruleengine/DefaultRuleEngineCallServiceTest.java @@ -132,7 +132,7 @@ public class DefaultRuleEngineCallServiceTest { private TransportProtos.RestApiCallResponseMsgProto getResponse(UUID requestId, TbMsg msg) { return TransportProtos.RestApiCallResponseMsgProto.newBuilder() - .setResponse(TbMsg.toByteString(msg)) + .setResponseProto(TbMsg.toProto(msg)) .setRequestIdMSB(requestId.getMostSignificantBits()) .setRequestIdLSB(requestId.getLeastSignificantBits()) .build(); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 47fb80c90e..88e7c6b691 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -371,11 +371,7 @@ public final class TbMsg implements Serializable { this.callback = Objects.requireNonNullElse(callback, TbMsgCallback.EMPTY); } - public static ByteString toByteString(TbMsg msg) { - return ByteString.copyFrom(toByteArray(msg)); - } - - public static byte[] toByteArray(TbMsg msg) { + public static MsgProtos.TbMsgProto toProto(TbMsg msg) { MsgProtos.TbMsgProto.Builder builder = MsgProtos.TbMsgProto.newBuilder(); builder.setId(msg.getId().toString()); builder.setTs(msg.getTs()); @@ -415,47 +411,55 @@ public final class TbMsg implements Serializable { } builder.setCtx(msg.ctx.toProto()); - return builder.build().toByteArray(); + return builder.build(); } - public static TbMsg fromBytes(String queueName, byte[] data, TbMsgCallback callback) { + //TODO: added for processing old messages from queue, should be removed after release + @Deprecated(forRemoval = true) + public static TbMsg fromProto(String queueName, MsgProtos.TbMsgProto proto, ByteString data, TbMsgCallback callback) { try { - MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(data); - TbMsgMetaData metaData = new TbMsgMetaData(proto.getMetaData().getDataMap()); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - CustomerId customerId = null; - RuleChainId ruleChainId = null; - RuleNodeId ruleNodeId = null; - UUID correlationId = null; - Integer partition = null; - if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) { - customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB())); - } - if (proto.getRuleChainIdMSB() != 0L && proto.getRuleChainIdLSB() != 0L) { - ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB())); - } - if (proto.getRuleNodeIdMSB() != 0L && proto.getRuleNodeIdLSB() != 0L) { - ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB())); - } - if (proto.getCorrelationIdMSB() != 0L && proto.getCorrelationIdLSB() != 0L) { - correlationId = new UUID(proto.getCorrelationIdMSB(), proto.getCorrelationIdLSB()); - partition = proto.getPartition(); + if (!data.isEmpty()) { + proto = MsgProtos.TbMsgProto.parseFrom(data); } - - TbMsgProcessingCtx ctx; - if (proto.hasCtx()) { - ctx = TbMsgProcessingCtx.fromProto(proto.getCtx()); - } else { - // Backward compatibility with unprocessed messages fetched from queue after update. - ctx = new TbMsgProcessingCtx(proto.getRuleNodeExecCounter()); - } - - TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; - return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), null, proto.getType(), entityId, customerId, - metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, ctx, callback); } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); } + return fromProto(queueName, proto, callback); + } + + public static TbMsg fromProto(String queueName, MsgProtos.TbMsgProto proto, TbMsgCallback callback) { + TbMsgMetaData metaData = new TbMsgMetaData(proto.getMetaData().getDataMap()); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + CustomerId customerId = null; + RuleChainId ruleChainId = null; + RuleNodeId ruleNodeId = null; + UUID correlationId = null; + Integer partition = null; + if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) { + customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB())); + } + if (proto.getRuleChainIdMSB() != 0L && proto.getRuleChainIdLSB() != 0L) { + ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB())); + } + if (proto.getRuleNodeIdMSB() != 0L && proto.getRuleNodeIdLSB() != 0L) { + ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB())); + } + if (proto.getCorrelationIdMSB() != 0L && proto.getCorrelationIdLSB() != 0L) { + correlationId = new UUID(proto.getCorrelationIdMSB(), proto.getCorrelationIdLSB()); + partition = proto.getPartition(); + } + + TbMsgProcessingCtx ctx; + if (proto.hasCtx()) { + ctx = TbMsgProcessingCtx.fromProto(proto.getCtx()); + } else { + // Backward compatibility with unprocessed messages fetched from queue after update. + ctx = new TbMsgProcessingCtx(proto.getRuleNodeExecCounter()); + } + + TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; + return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), null, proto.getType(), entityId, customerId, + metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, ctx, callback); } public TbMsg copyWithRuleChainId(RuleChainId ruleChainId) { diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 228a4039d2..ceef0914a7 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -20,6 +20,8 @@ package transport; option java_package = "org.thingsboard.server.gen.transport"; option java_outer_classname = "TransportProtos"; +import "tbmsg.proto"; + /** * Common data structures */ @@ -108,7 +110,8 @@ message SessionInfoProto { message RestApiCallResponseMsgProto { int64 requestIdMSB = 1; int64 requestIdLSB = 2; - bytes response = 5; + bytes response = 5 [deprecated = true]; + msgqueue.TbMsgProto responseProto = 6; } enum SessionEvent { @@ -1556,9 +1559,10 @@ message ToEdgeEventNotificationMsg { message ToRuleEngineMsg { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; - bytes tbMsg = 3; + bytes tbMsg = 3 [deprecated = true]; repeated string relationTypes = 4; string failureMessage = 5; + msgqueue.TbMsgProto tbMsgProto = 6; } message ToRuleEngineNotificationMsg { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbRuleEngineProducerService.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbRuleEngineProducerService.java index 49b40e3a6d..aefce4e176 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbRuleEngineProducerService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbRuleEngineProducerService.java @@ -65,7 +65,7 @@ public class TbRuleEngineProducerService { log.trace("[{}][{}] Pushing to topic {} message {}", tenantId, tbMsg.getOriginator(), tpi.getFullTopicName(), tbMsg); } ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() - .setTbMsg(TbMsg.toByteString(tbMsg)) + .setTbMsgProto(TbMsg.toProto(tbMsg)) .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); producer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback); From f87aa0b1425f03ce9ec8ce64b34a7ad6b0ff5466 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 28 Jan 2025 14:35:21 +0100 Subject: [PATCH 02/40] Added new relation between rule chains if RuleChainInput node is used --- .../common/data/relation/EntityRelation.java | 1 + .../server/dao/rule/BaseRuleChainService.java | 35 +++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java b/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java index 62b5caafd3..ff45c01500 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java @@ -41,6 +41,7 @@ public class EntityRelation implements HasVersion, Serializable { public static final String EDGE_TYPE = "ManagedByEdge"; public static final String CONTAINS_TYPE = "Contains"; public static final String MANAGES_TYPE = "Manages"; + public static final String USES_TYPE = "Uses"; @Setter private EntityId from; diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 0e51cf256a..1ccc5fa95f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -71,11 +71,13 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; @@ -196,11 +198,18 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC } } } - + RuleChainId ruleChainId = ruleChain.getId(); List updatedRuleNodes = new ArrayList<>(); List existingRuleNodes = getRuleChainNodes(tenantId, ruleChainMetaData.getRuleChainId()); for (RuleNode existingNode : existingRuleNodes) { relationService.deleteEntityRelations(tenantId, existingNode.getId()); + if (existingNode.getType().equals("org.thingsboard.rule.engine.flow.TbRuleChainInputNode")) { + if (existingNode.getConfiguration().has("ruleChainId")) { + RuleChainId targetRuleChainId = extractRuleChainIdFromInputNode(existingNode); + var relation = createRuleChainInputRelation(ruleChainId, targetRuleChainId); + relationService.deleteRelation(tenantId, relation); + } + } Integer index = ruleNodeIndexMap.get(existingNode.getId()); RuleNode newRuleNode = null; if (index != null) { @@ -212,7 +221,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC } updatedRuleNodes.add(new RuleNodeUpdateResult(existingNode, newRuleNode)); } - RuleChainId ruleChainId = ruleChain.getId(); + if (nodes != null) { long now = System.currentTimeMillis(); for (RuleNode node : toAddOrUpdate) { @@ -225,6 +234,13 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC RuleNode savedNode = ruleNodeDao.save(tenantId, node); relations.add(new EntityRelation(ruleChainMetaData.getRuleChainId(), savedNode.getId(), EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN)); + if (node.getType().equals("org.thingsboard.rule.engine.flow.TbRuleChainInputNode")) { + if (node.getConfiguration().has("ruleChainId")) { + RuleChainId targetRuleChainId = extractRuleChainIdFromInputNode(node); + var relation = createRuleChainInputRelation(ruleChainId, targetRuleChainId); + relations.add(relation); + } + } int index = nodes.indexOf(node); nodes.set(index, savedNode); ruleNodeIndexMap.put(savedNode.getId(), index); @@ -295,6 +311,21 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC return RuleChainUpdateResult.successful(updatedRuleNodes); } + private EntityRelation createRuleChainInputRelation(RuleChainId ruleChainId, RuleChainId targetRuleChainId) { + EntityRelation relation = new EntityRelation(); + relation.setFrom(ruleChainId); + relation.setTo(targetRuleChainId); + relation.setType(EntityRelation.USES_TYPE); + relation.setTypeGroup(RelationTypeGroup.COMMON); + return relation; + } + + private RuleChainId extractRuleChainIdFromInputNode(RuleNode node) { + JsonNode configuration = node.getConfiguration(); + UUID targetUuid = UUID.fromString(configuration.get("ruleChainId").asText()); + return new RuleChainId(targetUuid); + } + @Override public RuleChainMetaData loadRuleChainMetaData(TenantId tenantId, RuleChainId ruleChainId) { Validator.validateId(ruleChainId, "Incorrect rule chain id."); From f06fc5b27597500919545e09ea2b7b6d1a26b2d5 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 28 Jan 2025 14:38:35 +0100 Subject: [PATCH 03/40] imports optimization --- .../org/thingsboard/server/dao/rule/BaseRuleChainService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 1ccc5fa95f..eb78cf316e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -71,7 +71,6 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; From bbdeda95580c5903c1cff4981ed90671168c96c6 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Sun, 9 Mar 2025 19:48:38 +0100 Subject: [PATCH 04/40] added tests --- .../dao/service/RuleChainServiceTest.java | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/RuleChainServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/RuleChainServiceTest.java index 86b76bd5a5..8b5ebd4306 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/RuleChainServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/RuleChainServiceTest.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.service; import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.Assert; import org.junit.Test; import org.junit.jupiter.api.Assertions; @@ -28,12 +29,14 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.rule.RuleChainService; import java.io.IOException; @@ -44,6 +47,7 @@ import java.util.UUID; import java.util.function.Function; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.thingsboard.server.common.data.relation.EntityRelation.USES_TYPE; /** * Created by igor on 3/13/18. @@ -55,6 +59,8 @@ public class RuleChainServiceTest extends AbstractServiceTest { EdgeService edgeService; @Autowired RuleChainService ruleChainService; + @Autowired + RelationService relationService; private IdComparator idComparator = new IdComparator<>(); private IdComparator ruleNodeIdComparator = new IdComparator<>(); @@ -354,6 +360,66 @@ public class RuleChainServiceTest extends AbstractServiceTest { Assert.assertTrue(ruleChainById.isRoot()); } + @Test + public void testSaveRuleChainWithInputNode() { + RuleChain toRuleChain = new RuleChain(); + toRuleChain.setName("To Rule Chain"); + toRuleChain.setTenantId(tenantId); + RuleChain savedToRuleChain = ruleChainService.saveRuleChain(toRuleChain); + + RuleChain fromRuleChain = new RuleChain(); + fromRuleChain.setName("From RuleChain"); + fromRuleChain.setTenantId(tenantId); + RuleChain savedFromRuleChain = ruleChainService.saveRuleChain(fromRuleChain); + + RuleChainMetaData ruleChainMetaData = new RuleChainMetaData(); + ruleChainMetaData.setRuleChainId(savedFromRuleChain.getId()); + + RuleNode ruleNode = new RuleNode(); + ruleNode.setName("Input node"); + ruleNode.setType("org.thingsboard.rule.engine.flow.TbRuleChainInputNode"); + ObjectNode configuration = JacksonUtil.newObjectNode(); + configuration.put("ruleChainId", savedToRuleChain.getId().toString()); + ruleNode.setConfiguration(configuration); + + List ruleNodes = new ArrayList<>(); + ruleNodes.add(ruleNode); + ruleChainMetaData.setFirstNodeIndex(0); + ruleChainMetaData.setNodes(ruleNodes); + + ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData, Function.identity()); + + List relations = relationService.findByFromAndType(tenantId, savedFromRuleChain.getId(), USES_TYPE, RelationTypeGroup.COMMON); + Assert.assertEquals(1, relations.size()); + EntityRelation usesRelation = relations.get(0); + Assert.assertEquals(savedFromRuleChain.getId(), usesRelation.getFrom()); + Assert.assertEquals(savedToRuleChain.getId(), usesRelation.getTo()); + + RuleChain newToRuleChain = new RuleChain(); + newToRuleChain.setName("New To Rule Chain"); + newToRuleChain.setTenantId(tenantId); + RuleChain savedNewToRuleChain = ruleChainService.saveRuleChain(newToRuleChain); + + RuleNode newRuleNode = new RuleNode(); + newRuleNode.setName("Input node"); + newRuleNode.setType("org.thingsboard.rule.engine.flow.TbRuleChainInputNode"); + ObjectNode newConfiguration = JacksonUtil.newObjectNode(); + configuration.put("ruleChainId", savedNewToRuleChain.getId().toString()); + newRuleNode.setConfiguration(newConfiguration); + + List newRuleNodes = new ArrayList<>(); + newRuleNodes.add(newRuleNode); + RuleChainMetaData foundRuleChainMetaData = ruleChainService.loadRuleChainMetaData(tenantId, ruleChainMetaData.getRuleChainId()); + foundRuleChainMetaData.setNodes(newRuleNodes); + ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData, Function.identity()); + + List newRelations = relationService.findByFromAndType(tenantId, savedFromRuleChain.getId(), USES_TYPE, RelationTypeGroup.COMMON); + Assert.assertEquals(1, relations.size()); + EntityRelation newUsesRelation = newRelations.get(0); + Assert.assertEquals(savedFromRuleChain.getId(), newUsesRelation.getFrom()); + Assert.assertEquals(savedNewToRuleChain.getId(), newUsesRelation.getTo()); + } + private RuleChainId saveRuleChainAndSetAutoAssignToEdge(String name) { RuleChain edgeRuleChain = new RuleChain(); edgeRuleChain.setTenantId(tenantId); From 37482b51f33f7e4003ffddc534c98aa10c1f31e5 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 11 Mar 2025 12:33:05 +0100 Subject: [PATCH 05/40] added upgrade script --- .../update/DefaultDataUpdateService.java | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index e8622fae37..27e0f6adc2 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -24,23 +24,36 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.alarm.AlarmSeverity; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.query.DynamicValue; import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.sql.JpaExecutorService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.component.RuleNodeClassInfo; +import org.thingsboard.server.service.install.DbUpgradeExecutorService; import org.thingsboard.server.service.install.InstallScripts; import org.thingsboard.server.utils.TbNodeUpgradeUtils; import java.util.ArrayList; import java.util.List; +import java.util.UUID; import java.util.concurrent.ExecutionException; +import static org.thingsboard.server.common.data.relation.EntityRelation.USES_TYPE; + @Service @Profile("install") @Slf4j @@ -52,6 +65,12 @@ public class DefaultDataUpdateService implements DataUpdateService { @Autowired private RuleChainService ruleChainService; + @Autowired + private RelationService relationService; + + @Autowired + private TenantService tenantService; + @Autowired private ComponentDiscoveryService componentDiscoveryService; @@ -61,13 +80,66 @@ public class DefaultDataUpdateService implements DataUpdateService { @Autowired private InstallScripts installScripts; + @Autowired + private DbUpgradeExecutorService executorService; + @Override public void updateData() throws Exception { log.info("Updating data ..."); //TODO: should be cleaned after each release + inputNodesUpdater.updateEntities(); log.info("Data updated."); } + //TODO: should be removed after release + private final PaginatedUpdater inputNodesUpdater = new PaginatedUpdater<>() { + @Override + protected String getName() { + return "Input nodes updater"; + } + + @Override + protected PageData findEntities(String type, PageLink pageLink) { + return tenantService.findTenants(pageLink); + } + + @Override + protected void updateEntity(Tenant tenant) { + TenantId tenantId = tenant.getId(); + try { + var inputNodes = ruleChainService.findRuleNodesByTenantIdAndType(tenantId, "org.thingsboard.rule.engine.flow.TbRuleChainInputNode"); + var resultFutures = inputNodes.stream().map(ruleNode -> { + try { + JsonNode id = ruleNode.getConfiguration().get("ruleChainId"); + if (id != null) { + RuleChainId toRuleChainId = new RuleChainId(UUID.fromString(id.asText())); + RuleChainId fromRuleChainId = ruleNode.getRuleChainId(); + var isExistFuture = relationService.checkRelationAsync(null, fromRuleChainId, toRuleChainId, USES_TYPE, RelationTypeGroup.COMMON); + Futures.transformAsync(isExistFuture, isExist -> { + if (!isExist) { + EntityRelation relation = new EntityRelation(); + relation.setFrom(fromRuleChainId); + relation.setTo(toRuleChainId); + relation.setType(EntityRelation.USES_TYPE); + relation.setTypeGroup(RelationTypeGroup.COMMON); + return relationService.saveRelationAsync(tenantId, relation); + } + return Futures.immediateFuture(null); + }, executorService); + } + } catch (Exception e) { + log.error("[{}] Create relation for input node: [{}]", tenantId, ruleNode, e); + } + return Futures.immediateFuture(null); + }).toList(); + + Futures.allAsList(resultFutures).get(); + } catch (Exception e) { + log.error("[{}] Unable to update Tenant input nodes", tenantId, e); + } + } + }; + @Override public void upgradeRuleNodes() { int totalRuleNodesUpgraded = 0; From ccd970b0da44092d8adbc817080968ad503dc817 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 12 Mar 2025 17:53:00 +0200 Subject: [PATCH 06/40] fixed resource vc restoring --- .../ie/importing/impl/AssetImportService.java | 2 +- .../impl/AssetProfileImportService.java | 2 +- .../impl/BaseEntityImportService.java | 33 ++++++++++---- .../importing/impl/CustomerImportService.java | 2 +- .../impl/DashboardImportService.java | 8 ++-- .../importing/impl/DeviceImportService.java | 2 +- .../impl/DeviceProfileImportService.java | 2 +- .../impl/EntityViewImportService.java | 2 +- .../impl/NotificationRuleImportService.java | 2 +- .../impl/NotificationTargetImportService.java | 2 +- .../NotificationTemplateImportService.java | 2 +- .../importing/impl/ResourceImportService.java | 11 ++--- .../impl/RuleChainImportService.java | 12 ++--- .../impl/WidgetTypeImportService.java | 6 +-- .../impl/WidgetsBundleImportService.java | 6 +-- .../DefaultEntitiesVersionControlService.java | 2 +- .../server/utils/LwM2mObjectModelUtils.java | 1 - .../server/common/data/TbResourceInfo.java | 45 +++++++++++++++++++ .../common/data/sync/vc/EntityLoadError.java | 6 ++- 19 files changed, 108 insertions(+), 40 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java index 7cd4c3aca1..9bf44ffbec 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java @@ -46,7 +46,7 @@ public class AssetImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected Asset saveOrUpdate(EntitiesImportCtx ctx, Asset asset, EntityExportData exportData, IdProvider idProvider, CompareResult compareResult) { Asset savedAsset = assetService.saveAsset(asset); if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) { importCalculatedFields(ctx, savedAsset, exportData, idProvider); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java index 32a0090a4a..d5663c34a4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java @@ -49,7 +49,7 @@ public class AssetProfileImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected AssetProfile saveOrUpdate(EntitiesImportCtx ctx, AssetProfile assetProfile, EntityExportData exportData, IdProvider idProvider, CompareResult compareResult) { AssetProfile saved = assetProfileService.saveAssetProfile(assetProfile); if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) { importCalculatedFields(ctx, saved, exportData, idProvider); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index bfa95af83c..83052360b6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -18,6 +18,8 @@ package org.thingsboard.server.service.sync.ie.importing.impl; import com.fasterxml.jackson.databind.JsonNode; import com.google.api.client.util.Objects; import com.google.common.util.concurrent.FutureCallback; +import lombok.AllArgsConstructor; +import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.Nullable; @@ -117,10 +119,10 @@ public abstract class BaseEntityImportService importResult, D exportData, IdProvider idProvider) throws ThingsboardException { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java index 8774c77870..d4179b639d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java @@ -52,7 +52,7 @@ public class CustomerImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected Customer saveOrUpdate(EntitiesImportCtx ctx, Customer customer, EntityExportData exportData, IdProvider idProvider, CompareResult compareResult) { if (!customer.isPublic()) { return customerService.saveCustomer(customer); } else { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index e9407f2d58..6744b04f55 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -75,7 +75,7 @@ public class DashboardImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected Dashboard saveOrUpdate(EntitiesImportCtx ctx, Dashboard dashboard, EntityExportData exportData, IdProvider idProvider, CompareResult compareResult) { var tenantId = ctx.getTenantId(); Set assignedCustomers = Optional.ofNullable(dashboard.getAssignedCustomers()).orElse(Collections.emptySet()).stream() @@ -116,8 +116,10 @@ public class DashboardImportService extends BaseEntityImportService exportData, Dashboard prepared, Dashboard existing) { - return super.compare(ctx, exportData, prepared, existing) || !prepared.getConfiguration().equals(existing.getConfiguration()); + protected CompareResult compare(EntitiesImportCtx ctx, EntityExportData exportData, Dashboard prepared, Dashboard existing) { + CompareResult result = super.compare(ctx, exportData, prepared, existing); + result.setNeedUpdate(result.isNeedUpdate() || !prepared.getConfiguration().equals(existing.getConfiguration())); + return result; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java index 84e264efdd..4ace9ff938 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java @@ -63,7 +63,7 @@ public class DeviceImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected DeviceProfile saveOrUpdate(EntitiesImportCtx ctx, DeviceProfile deviceProfile, EntityExportData exportData, IdProvider idProvider, CompareResult compareResult) { DeviceProfile saved = deviceProfileService.saveDeviceProfile(deviceProfile); if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) { importCalculatedFields(ctx, saved, exportData, idProvider); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java index 8e8f2e90a6..1479943b08 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java @@ -55,7 +55,7 @@ public class EntityViewImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected EntityView saveOrUpdate(EntitiesImportCtx ctx, EntityView entityView, EntityExportData exportData, IdProvider idProvider, CompareResult compareResult) { return entityViewService.saveEntityView(entityView); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationRuleImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationRuleImportService.java index 52f91912e6..a2ae7c8a13 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationRuleImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationRuleImportService.java @@ -135,7 +135,7 @@ public class NotificationRuleImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected NotificationRule saveOrUpdate(EntitiesImportCtx ctx, NotificationRule notificationRule, EntityExportData exportData, IdProvider idProvider, CompareResult compareResult) { ConstraintValidator.validateFields(notificationRule); return notificationRuleService.saveNotificationRule(ctx.getTenantId(), notificationRule); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationTargetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationTargetImportService.java index 9bca0f8054..4323aba9cc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationTargetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationTargetImportService.java @@ -80,7 +80,7 @@ public class NotificationTargetImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected NotificationTarget saveOrUpdate(EntitiesImportCtx ctx, NotificationTarget notificationTarget, EntityExportData exportData, IdProvider idProvider, CompareResult compareResult) { ConstraintValidator.validateFields(notificationTarget); return notificationTargetService.saveNotificationTarget(ctx.getTenantId(), notificationTarget); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationTemplateImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationTemplateImportService.java index 1452321744..09a93e0937 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationTemplateImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationTemplateImportService.java @@ -48,7 +48,7 @@ public class NotificationTemplateImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected NotificationTemplate saveOrUpdate(EntitiesImportCtx ctx, NotificationTemplate notificationTemplate, EntityExportData exportData, IdProvider idProvider, CompareResult compareResult) { ConstraintValidator.validateFields(notificationTemplate); return notificationTemplateService.saveNotificationTemplate(ctx.getTenantId(), notificationTemplate); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java index bedda621e0..e97a5ad06e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java @@ -58,17 +58,18 @@ public class ResourceImportService extends BaseEntityImportService exportData, TbResource prepared, TbResource existing) { - return true; + protected TbResource deepCopy(TbResource resource) { + return new TbResource(resource); } @Override - protected TbResource deepCopy(TbResource resource) { - return new TbResource(resource); + protected void cleanupForComparison(TbResource resource) { + super.cleanupForComparison(resource); + resource.setSearchText(null); } @Override - protected TbResource saveOrUpdate(EntitiesImportCtx ctx, TbResource resource, EntityExportData exportData, IdProvider idProvider) { + protected TbResource saveOrUpdate(EntitiesImportCtx ctx, TbResource resource, EntityExportData exportData, IdProvider idProvider, CompareResult compareResult) { if (resource.getResourceType() == ResourceType.IMAGE) { return new TbResource(imageService.saveImage(resource)); } else { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index a7c68af297..6ad2b98953 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -103,7 +103,7 @@ public class RuleChainImportService extends BaseEntityImportService { String bundleAlias = widgetTypeNode.remove("bundleAlias").asText(); @@ -75,8 +75,8 @@ public class WidgetsBundleImportService extends BaseEntityImportService analyze(Throwable e, EntityId externalId) { diff --git a/application/src/main/java/org/thingsboard/server/utils/LwM2mObjectModelUtils.java b/application/src/main/java/org/thingsboard/server/utils/LwM2mObjectModelUtils.java index fb6125bc0b..b1c71c9ded 100644 --- a/application/src/main/java/org/thingsboard/server/utils/LwM2mObjectModelUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/LwM2mObjectModelUtils.java @@ -54,7 +54,6 @@ public class LwM2mObjectModelUtils { if (resource.getId() == null) { resource.setTitle(name + " id=" + objectModel.id + " v" + objectModel.version); } - resource.setSearchText(resourceKey + LWM2M_SEPARATOR_SEARCH_TEXT + name); } else { throw new DataValidationException(String.format("Could not parse the XML of objectModel with name %s", resource.getSearchText())); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TbResourceInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/TbResourceInfo.java index a3bf383503..89f5fa38b0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/TbResourceInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TbResourceInfo.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; +import java.util.Objects; import java.util.function.UnaryOperator; @Schema @@ -151,4 +152,48 @@ public class TbResourceInfo extends BaseData implements HasName, H this.descriptor = value != null ? mapper.valueToTree(value) : null; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + + TbResourceInfo that = (TbResourceInfo) o; + + if (isPublic != that.isPublic) return false; + if (!Objects.equals(tenantId, that.tenantId)) return false; + if (!Objects.equals(title, that.title)) return false; + if (resourceType != that.resourceType) return false; + if (resourceSubType != that.resourceSubType) return false; + if (!Objects.equals(resourceKey, that.resourceKey)) return false; + if (!Objects.equals(publicResourceKey, that.publicResourceKey)) + return false; + if (!Objects.equals(searchText, that.searchText)) return false; + if (!Objects.equals(etag, that.etag)) return false; + if (!Objects.equals(fileName, that.fileName)) return false; + if (!Objects.equals(descriptor, that.descriptor)) { + if (!((descriptor == null || descriptor.isNull()) && (that.descriptor == null || that.descriptor.isNull()))){ + return false; + } + } + return Objects.equals(externalId, that.externalId); + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (tenantId != null ? tenantId.hashCode() : 0); + result = 31 * result + (title != null ? title.hashCode() : 0); + result = 31 * result + (resourceType != null ? resourceType.hashCode() : 0); + result = 31 * result + (resourceSubType != null ? resourceSubType.hashCode() : 0); + result = 31 * result + (resourceKey != null ? resourceKey.hashCode() : 0); + result = 31 * result + (isPublic ? 1 : 0); + result = 31 * result + (publicResourceKey != null ? publicResourceKey.hashCode() : 0); + result = 31 * result + (searchText != null ? searchText.hashCode() : 0); + result = 31 * result + (etag != null ? etag.hashCode() : 0); + result = 31 * result + (fileName != null ? fileName.hashCode() : 0); + result = 31 * result + (descriptor != null ? descriptor.hashCode() : 0); + result = 31 * result + (externalId != null ? externalId.hashCode() : 0); + return result; + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java index 049a7e5d2a..157f3bf553 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java @@ -45,11 +45,15 @@ public class EntityLoadError implements Serializable { } public static EntityLoadError runtimeError(Throwable e) { + return runtimeError(e, null); + } + + public static EntityLoadError runtimeError(Throwable e, EntityId externalId) { String message = e.getMessage(); if (StringUtils.isEmpty(message)) { message = "unexpected error (" + ClassUtils.getShortClassName(e.getClass()) + ")"; } - return EntityLoadError.builder().type("RUNTIME").message(message).build(); + return EntityLoadError.builder().type("RUNTIME").message(message).source(externalId).build(); } } From 29e9a3d122e07de10916980c0fa79cf3a6699850 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 12 Mar 2025 18:31:30 +0200 Subject: [PATCH 07/40] added test --- .../importing/impl/ResourceImportService.java | 6 ++- .../service/sync/vc/VersionControlTest.java | 44 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java index e97a5ad06e..f5ca15cfca 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java @@ -73,7 +73,11 @@ public class ResourceImportService extends BaseEntityImportService & HasTenantId> void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, E importedEntity) { assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1); assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2); @@ -1126,4 +1152,22 @@ public class VersionControlTest extends AbstractControllerTest { return doGetTypedWithPageLink("/api/" + entityId.getEntityType() + "/" + entityId.getId() + "/calculatedFields?", new TypeReference>() {}, new PageLink(100, 0)).getData(); } + private TbResourceInfo createResource(String name) { + TbResource resource = new TbResource(); + resource.setResourceType(ResourceType.JKS); + resource.setTitle(name); + resource.setFileName(DEFAULT_FILE_NAME); + resource.setEncodedData(TEST_DATA); + + return saveTbResource(resource); + } + + private TbResourceInfo saveTbResource(TbResource tbResource) { + return doPost("/api/resource", tbResource, TbResourceInfo.class); + } + + private TbResource findResource(String name) throws Exception { + return doGetTypedWithPageLink("/api/resource?", new TypeReference>() {}, new PageLink(100, 0, name)).getData().get(0); + } + } From e1dfa3f80329dd3a29d70ddc223c5d9036c512f3 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 12 Mar 2025 19:24:28 +0200 Subject: [PATCH 08/40] refactoring --- .../impl/BaseEntityImportService.java | 28 +++++++++++-------- .../impl/DashboardImportService.java | 2 +- .../impl/RuleChainImportService.java | 4 +-- .../impl/WidgetTypeImportService.java | 2 +- .../impl/WidgetsBundleImportService.java | 2 +- .../controller/TbResourceControllerTest.java | 4 +-- .../service/sync/vc/VersionControlTest.java | 5 ++-- 7 files changed, 25 insertions(+), 22 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 83052360b6..27e4209e7c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -121,7 +121,7 @@ public abstract class BaseEntityImportService exportData, Dashboard prepared, Dashboard existing) { CompareResult result = super.compare(ctx, exportData, prepared, existing); - result.setNeedUpdate(result.isNeedUpdate() || !prepared.getConfiguration().equals(existing.getConfiguration())); + result.setUpdateNeeded(result.isUpdateNeeded() || !prepared.getConfiguration().equals(existing.getConfiguration())); return result; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index 6ad2b98953..f5b4521629 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -117,11 +117,11 @@ public class RuleChainImportService extends BaseEntityImportService Date: Thu, 13 Mar 2025 18:33:56 +0200 Subject: [PATCH 09/40] compare method refactoring --- .../importing/impl/BaseEntityImportService.java | 16 ++++++++++++---- .../importing/impl/DashboardImportService.java | 6 ++---- .../importing/impl/RuleChainImportService.java | 12 +++++++----- .../server/utils/LwM2mObjectModelUtils.java | 1 + .../service/sync/vc/VersionControlTest.java | 4 ++-- .../server/common/data/TbResourceInfo.java | 2 +- 6 files changed, 25 insertions(+), 16 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 27e4209e7c..57b4737be6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -141,7 +141,7 @@ public abstract class BaseEntityImportService exportData, Dashboard prepared, Dashboard existing) { - CompareResult result = super.compare(ctx, exportData, prepared, existing); - result.setUpdateNeeded(result.isUpdateNeeded() || !prepared.getConfiguration().equals(existing.getConfiguration())); - return result; + protected boolean isUpdateNeeded(EntitiesImportCtx ctx, EntityExportData exportData, Dashboard prepared, Dashboard existing) { + return super.isUpdateNeeded(ctx, exportData, prepared, existing) || !prepared.getConfiguration().equals(existing.getConfiguration()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index f5b4521629..baf1b84f82 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.audit.ActionType; @@ -27,6 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.rule.RuleNodeDao; @@ -115,15 +117,15 @@ public class RuleChainImportService extends BaseEntityImportService implements HasName, H if (!Objects.equals(resourceKey, that.resourceKey)) return false; if (!Objects.equals(publicResourceKey, that.publicResourceKey)) return false; - if (!Objects.equals(searchText, that.searchText)) return false; + if (!Objects.equals(getSearchText(), that.getSearchText())) return false; if (!Objects.equals(etag, that.etag)) return false; if (!Objects.equals(fileName, that.fileName)) return false; if (!Objects.equals(descriptor, that.descriptor)) { From 753071ea17fbb4eb74af7d88839f525f459683b8 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 14 Mar 2025 14:48:32 +0100 Subject: [PATCH 10/40] minor refactoring --- .../update/DefaultDataUpdateService.java | 24 +++++++------------ .../server/dao/rule/BaseRuleChainService.java | 6 ++--- .../dao/service/RuleChainServiceTest.java | 5 ++-- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index b1f80cb793..4c90258330 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -49,7 +49,7 @@ import java.util.List; import java.util.UUID; import java.util.concurrent.ExecutionException; -import static org.thingsboard.server.common.data.relation.EntityRelation.USES_TYPE; +import static org.thingsboard.server.dao.rule.BaseRuleChainService.TB_RULE_CHAIN_INPUT_NODE; @Service @Profile("install") @@ -98,28 +98,22 @@ public class DefaultDataUpdateService implements DataUpdateService { protected void updateEntity(Tenant tenant) { TenantId tenantId = tenant.getId(); try { - var inputNodes = ruleChainService.findRuleNodesByTenantIdAndType(tenantId, "org.thingsboard.rule.engine.flow.TbRuleChainInputNode"); + var inputNodes = ruleChainService.findRuleNodesByTenantIdAndType(tenantId, TB_RULE_CHAIN_INPUT_NODE); var resultFutures = inputNodes.stream().map(ruleNode -> { try { JsonNode id = ruleNode.getConfiguration().get("ruleChainId"); if (id != null) { RuleChainId toRuleChainId = new RuleChainId(UUID.fromString(id.asText())); RuleChainId fromRuleChainId = ruleNode.getRuleChainId(); - var isExistFuture = relationService.checkRelationAsync(null, fromRuleChainId, toRuleChainId, USES_TYPE, RelationTypeGroup.COMMON); - Futures.transformAsync(isExistFuture, isExist -> { - if (!isExist) { - EntityRelation relation = new EntityRelation(); - relation.setFrom(fromRuleChainId); - relation.setTo(toRuleChainId); - relation.setType(EntityRelation.USES_TYPE); - relation.setTypeGroup(RelationTypeGroup.COMMON); - return relationService.saveRelationAsync(tenantId, relation); - } - return Futures.immediateFuture(null); - }, executorService); + EntityRelation relation = new EntityRelation(); + relation.setFrom(fromRuleChainId); + relation.setTo(toRuleChainId); + relation.setType(EntityRelation.USES_TYPE); + relation.setTypeGroup(RelationTypeGroup.COMMON); + return relationService.saveRelationAsync(tenantId, relation); } } catch (Exception e) { - log.error("[{}] Create relation for input node: [{}]", tenantId, ruleNode, e); + log.error("[{}] Failed to save relation for input node: [{}]", tenantId, ruleNode, e); } return Futures.immediateFuture(null); }).toList(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 9d61408199..1ce9bd555b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -210,7 +210,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC List existingRuleNodes = getRuleChainNodes(tenantId, ruleChainMetaData.getRuleChainId()); for (RuleNode existingNode : existingRuleNodes) { relationService.deleteEntityRelations(tenantId, existingNode.getId()); - if (existingNode.getType().equals("org.thingsboard.rule.engine.flow.TbRuleChainInputNode")) { + if (existingNode.getType().equals(TB_RULE_CHAIN_INPUT_NODE)) { if (existingNode.getConfiguration().has("ruleChainId")) { RuleChainId targetRuleChainId = extractRuleChainIdFromInputNode(existingNode); var relation = createRuleChainInputRelation(ruleChainId, targetRuleChainId); @@ -241,7 +241,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC RuleNode savedNode = ruleNodeDao.save(tenantId, node); relations.add(new EntityRelation(ruleChainMetaData.getRuleChainId(), savedNode.getId(), EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN)); - if (node.getType().equals("org.thingsboard.rule.engine.flow.TbRuleChainInputNode")) { + if (node.getType().equals(TB_RULE_CHAIN_INPUT_NODE)) { if (node.getConfiguration().has("ruleChainId")) { RuleChainId targetRuleChainId = extractRuleChainIdFromInputNode(node); var relation = createRuleChainInputRelation(ruleChainId, targetRuleChainId); @@ -280,7 +280,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC RuleNode targetNode = new RuleNode(); targetNode.setName(targetRuleChain != null ? targetRuleChain.getName() : "Rule Chain Input"); targetNode.setRuleChainId(ruleChainId); - targetNode.setType("org.thingsboard.rule.engine.flow.TbRuleChainInputNode"); + targetNode.setType(TB_RULE_CHAIN_INPUT_NODE); var configuration = JacksonUtil.newObjectNode(); configuration.put("ruleChainId", targetRuleChainId.getId().toString()); targetNode.setConfiguration(configuration); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/RuleChainServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/RuleChainServiceTest.java index 54144d5192..a2ca663619 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/RuleChainServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/RuleChainServiceTest.java @@ -48,6 +48,7 @@ import java.util.function.Function; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.thingsboard.server.common.data.relation.EntityRelation.USES_TYPE; +import static org.thingsboard.server.dao.rule.BaseRuleChainService.TB_RULE_CHAIN_INPUT_NODE; /** * Created by igor on 3/13/18. @@ -377,7 +378,7 @@ public class RuleChainServiceTest extends AbstractServiceTest { RuleNode ruleNode = new RuleNode(); ruleNode.setName("Input node"); - ruleNode.setType("org.thingsboard.rule.engine.flow.TbRuleChainInputNode"); + ruleNode.setType(TB_RULE_CHAIN_INPUT_NODE); ObjectNode configuration = JacksonUtil.newObjectNode(); configuration.put("ruleChainId", savedToRuleChain.getId().toString()); ruleNode.setConfiguration(configuration); @@ -402,7 +403,7 @@ public class RuleChainServiceTest extends AbstractServiceTest { RuleNode newRuleNode = new RuleNode(); newRuleNode.setName("Input node"); - newRuleNode.setType("org.thingsboard.rule.engine.flow.TbRuleChainInputNode"); + newRuleNode.setType(TB_RULE_CHAIN_INPUT_NODE); ObjectNode newConfiguration = JacksonUtil.newObjectNode(); configuration.put("ruleChainId", savedNewToRuleChain.getId().toString()); newRuleNode.setConfiguration(newConfiguration); From 8813e32910790330ae7c4765f3876dc8866d958c Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 27 Mar 2025 11:59:49 +0200 Subject: [PATCH 11/40] moved resource vc comparison fix to cleanupForComparison method --- .../importing/impl/ResourceImportService.java | 3 ++ .../server/common/data/TbResourceInfo.java | 45 ------------------- 2 files changed, 3 insertions(+), 45 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java index f5ca15cfca..96fcdd5425 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java @@ -66,6 +66,9 @@ public class ResourceImportService extends BaseEntityImportService implements HasName, H this.descriptor = value != null ? mapper.valueToTree(value) : null; } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; - - TbResourceInfo that = (TbResourceInfo) o; - - if (isPublic != that.isPublic) return false; - if (!Objects.equals(tenantId, that.tenantId)) return false; - if (!Objects.equals(title, that.title)) return false; - if (resourceType != that.resourceType) return false; - if (resourceSubType != that.resourceSubType) return false; - if (!Objects.equals(resourceKey, that.resourceKey)) return false; - if (!Objects.equals(publicResourceKey, that.publicResourceKey)) - return false; - if (!Objects.equals(getSearchText(), that.getSearchText())) return false; - if (!Objects.equals(etag, that.etag)) return false; - if (!Objects.equals(fileName, that.fileName)) return false; - if (!Objects.equals(descriptor, that.descriptor)) { - if (!((descriptor == null || descriptor.isNull()) && (that.descriptor == null || that.descriptor.isNull()))){ - return false; - } - } - return Objects.equals(externalId, that.externalId); - } - - @Override - public int hashCode() { - int result = super.hashCode(); - result = 31 * result + (tenantId != null ? tenantId.hashCode() : 0); - result = 31 * result + (title != null ? title.hashCode() : 0); - result = 31 * result + (resourceType != null ? resourceType.hashCode() : 0); - result = 31 * result + (resourceSubType != null ? resourceSubType.hashCode() : 0); - result = 31 * result + (resourceKey != null ? resourceKey.hashCode() : 0); - result = 31 * result + (isPublic ? 1 : 0); - result = 31 * result + (publicResourceKey != null ? publicResourceKey.hashCode() : 0); - result = 31 * result + (searchText != null ? searchText.hashCode() : 0); - result = 31 * result + (etag != null ? etag.hashCode() : 0); - result = 31 * result + (fileName != null ? fileName.hashCode() : 0); - result = 31 * result + (descriptor != null ? descriptor.hashCode() : 0); - result = 31 * result + (externalId != null ? externalId.hashCode() : 0); - return result; - } } From a5ce56fe4c0c53d5004221aef7bc2297e6718a39 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 28 Mar 2025 18:49:45 +0200 Subject: [PATCH 12/40] removed check for timeseries keys existence --- .../rule/engine/profile/AlarmRuleState.java | 6 - .../profile/TbDeviceProfileNodeTest.java | 210 ++++++++++++------ 2 files changed, 145 insertions(+), 71 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java index 81c8f8154c..a2a714a6df 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java @@ -88,12 +88,6 @@ class AlarmRuleState { } public boolean validateAttrUpdate(Set changedKeys) { - //If the attribute was updated, but no new telemetry arrived - we ignore this until new telemetry is there. - for (AlarmConditionFilterKey key : entityKeys) { - if (key.getType().equals(AlarmConditionKeyType.TIME_SERIES)) { - return false; - } - } for (AlarmConditionFilterKey key : changedKeys) { if (entityKeys.contains(key)) { return true; diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java index 7ecb1b2ad8..5163e21c05 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java @@ -58,6 +58,8 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntryAggWrapper; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.query.BooleanFilterPredicate; import org.thingsboard.server.common.data.query.DynamicValue; @@ -65,6 +67,7 @@ import org.thingsboard.server.common.data.query.DynamicValueSourceType; import org.thingsboard.server.common.data.query.EntityKeyValueType; import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.NumericFilterPredicate; +import org.thingsboard.server.common.data.query.NumericFilterPredicate.NumericOperation; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; @@ -72,6 +75,7 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; +import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import org.thingsboard.server.dao.timeseries.TimeseriesService; import java.math.BigDecimal; @@ -81,15 +85,22 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType.ATTRIBUTE; +import static org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType.TIME_SERIES; +import static org.thingsboard.server.common.data.query.NumericFilterPredicate.NumericOperation.GREATER; +import static org.thingsboard.server.common.data.query.NumericFilterPredicate.NumericOperation.LESS; @ExtendWith(MockitoExtension.class) public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { @@ -170,32 +181,16 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { DeviceProfile deviceProfile = new DeviceProfile(); DeviceProfileData deviceProfileData = new DeviceProfileData(); - AlarmConditionFilter highTempFilter = new AlarmConditionFilter(); - highTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); - highTempFilter.setValueType(EntityKeyValueType.NUMERIC); - NumericFilterPredicate highTemperaturePredicate = new NumericFilterPredicate(); - highTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); - highTemperaturePredicate.setValue(new FilterPredicateValue<>(30.0)); - highTempFilter.setPredicate(highTemperaturePredicate); - AlarmCondition alarmCondition = new AlarmCondition(); - alarmCondition.setCondition(Collections.singletonList(highTempFilter)); + AlarmCondition alarmCreateCondition = getNumericAlarmCondition(TIME_SERIES, "temperature", GREATER, 30.0); AlarmRule alarmRule = new AlarmRule(); - alarmRule.setCondition(alarmCondition); + alarmRule.setCondition(alarmCreateCondition); DeviceProfileAlarm dpa = new DeviceProfileAlarm(); dpa.setId("highTemperatureAlarmID"); dpa.setAlarmType("highTemperatureAlarm"); dpa.setCreateRules(new TreeMap<>(Collections.singletonMap(AlarmSeverity.CRITICAL, alarmRule))); - AlarmConditionFilter lowTempFilter = new AlarmConditionFilter(); - lowTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); - lowTempFilter.setValueType(EntityKeyValueType.NUMERIC); - NumericFilterPredicate lowTemperaturePredicate = new NumericFilterPredicate(); - lowTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS); - lowTemperaturePredicate.setValue(new FilterPredicateValue<>(10.0)); - lowTempFilter.setPredicate(lowTemperaturePredicate); AlarmRule clearRule = new AlarmRule(); - AlarmCondition clearCondition = new AlarmCondition(); - clearCondition.setCondition(Collections.singletonList(lowTempFilter)); + AlarmCondition clearCondition = getNumericAlarmCondition(TIME_SERIES, "temperature", LESS, 10.0); clearRule.setCondition(clearCondition); dpa.setClearRule(clearRule); @@ -261,25 +256,11 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { DeviceProfile deviceProfile = new DeviceProfile(); DeviceProfileData deviceProfileData = new DeviceProfileData(); - AlarmConditionFilter tempFilter = new AlarmConditionFilter(); - tempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); - tempFilter.setValueType(EntityKeyValueType.NUMERIC); - NumericFilterPredicate temperaturePredicate = new NumericFilterPredicate(); - temperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); - temperaturePredicate.setValue(new FilterPredicateValue<>(30.0)); - tempFilter.setPredicate(temperaturePredicate); - AlarmCondition alarmTempCondition = new AlarmCondition(); - alarmTempCondition.setCondition(Collections.singletonList(tempFilter)); + AlarmCondition alarmTempCondition = getNumericAlarmCondition(TIME_SERIES, "temperature", GREATER, 30.0); AlarmRule alarmTempRule = new AlarmRule(); alarmTempRule.setCondition(alarmTempCondition); - AlarmConditionFilter highTempFilter = new AlarmConditionFilter(); - highTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); - highTempFilter.setValueType(EntityKeyValueType.NUMERIC); - NumericFilterPredicate highTemperaturePredicate = new NumericFilterPredicate(); - highTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); - highTemperaturePredicate.setValue(new FilterPredicateValue<>(50.0)); - highTempFilter.setPredicate(highTemperaturePredicate); + AlarmConditionFilter highTempFilter = getAlarmConditionFilter(TIME_SERIES, "temperature", GREATER, 50.0); AlarmCondition alarmHighTempCondition = new AlarmCondition(); alarmHighTempCondition.setCondition(Collections.singletonList(highTempFilter)); AlarmRule alarmHighTempRule = new AlarmRule(); @@ -401,10 +382,10 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { alarmEnabledFilter.setPredicate(alarmEnabledPredicate); AlarmConditionFilter temperatureFilter = new AlarmConditionFilter(); - temperatureFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + temperatureFilter.setKey(new AlarmConditionFilterKey(TIME_SERIES, "temperature")); temperatureFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate temperaturePredicate = new NumericFilterPredicate(); - temperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + temperaturePredicate.setOperation(GREATER); temperaturePredicate.setValue(new FilterPredicateValue<>(20.0, null, null)); temperatureFilter.setPredicate(temperaturePredicate); @@ -494,10 +475,10 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { alarmEnabledFilter.setPredicate(alarmEnabledPredicate); AlarmConditionFilter temperatureFilter = new AlarmConditionFilter(); - temperatureFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + temperatureFilter.setKey(new AlarmConditionFilterKey(TIME_SERIES, "temperature")); temperatureFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate temperaturePredicate = new NumericFilterPredicate(); - temperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + temperaturePredicate.setOperation(GREATER); temperaturePredicate.setValue(new FilterPredicateValue<>(20.0, null, null)); temperatureFilter.setPredicate(temperaturePredicate); @@ -576,10 +557,10 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { Futures.immediateFuture(Collections.singletonList(entry)); AlarmConditionFilter highTempFilter = new AlarmConditionFilter(); - highTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + highTempFilter.setKey(new AlarmConditionFilterKey(TIME_SERIES, "temperature")); highTempFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate highTemperaturePredicate = new NumericFilterPredicate(); - highTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + highTemperaturePredicate.setOperation(GREATER); highTemperaturePredicate.setValue(new FilterPredicateValue<>( 0.0, null, @@ -670,10 +651,10 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { Futures.immediateFuture(Arrays.asList(entry, alarmDelayAttributeKvEntry)); AlarmConditionFilter highTempFilter = new AlarmConditionFilter(); - highTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + highTempFilter.setKey(new AlarmConditionFilterKey(TIME_SERIES, "temperature")); highTempFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate highTemperaturePredicate = new NumericFilterPredicate(); - highTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + highTemperaturePredicate.setOperation(GREATER); highTemperaturePredicate.setValue(new FilterPredicateValue<>( 0.0, null, @@ -805,10 +786,10 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { Futures.immediateFuture(Optional.empty()); AlarmConditionFilter highTempFilter = new AlarmConditionFilter(); - highTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + highTempFilter.setKey(new AlarmConditionFilterKey(TIME_SERIES, "temperature")); highTempFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate highTemperaturePredicate = new NumericFilterPredicate(); - highTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + highTemperaturePredicate.setOperation(GREATER); highTemperaturePredicate.setValue(new FilterPredicateValue<>( 0.0, null, @@ -937,10 +918,10 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { Futures.immediateFuture(Arrays.asList(entry, alarmDelayAttributeKvEntry)); AlarmConditionFilter highTempFilter = new AlarmConditionFilter(); - highTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + highTempFilter.setKey(new AlarmConditionFilterKey(TIME_SERIES, "temperature")); highTempFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate highTemperaturePredicate = new NumericFilterPredicate(); - highTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + highTemperaturePredicate.setOperation(GREATER); highTemperaturePredicate.setValue(new FilterPredicateValue<>( 0.0, null, @@ -1065,10 +1046,10 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { Futures.immediateFuture(Optional.empty()); AlarmConditionFilter highTempFilter = new AlarmConditionFilter(); - highTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + highTempFilter.setKey(new AlarmConditionFilterKey(TIME_SERIES, "temperature")); highTempFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate highTemperaturePredicate = new NumericFilterPredicate(); - highTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + highTemperaturePredicate.setOperation(GREATER); highTemperaturePredicate.setValue(new FilterPredicateValue<>( 0.0, null, @@ -1182,10 +1163,10 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { Futures.immediateFuture(Collections.singletonList(entry)); AlarmConditionFilter highTempFilter = new AlarmConditionFilter(); - highTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + highTempFilter.setKey(new AlarmConditionFilterKey(TIME_SERIES, "temperature")); highTempFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate highTemperaturePredicate = new NumericFilterPredicate(); - highTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + highTemperaturePredicate.setOperation(GREATER); highTemperaturePredicate.setValue(new FilterPredicateValue<>( 0.0, null, @@ -1299,10 +1280,10 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { Futures.immediateFuture(Collections.singletonList(entry)); AlarmConditionFilter highTempFilter = new AlarmConditionFilter(); - highTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + highTempFilter.setKey(new AlarmConditionFilterKey(TIME_SERIES, "temperature")); highTempFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate highTemperaturePredicate = new NumericFilterPredicate(); - highTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + highTemperaturePredicate.setOperation(GREATER); highTemperaturePredicate.setValue(new FilterPredicateValue<>( 0.0, null, @@ -1395,10 +1376,10 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { Futures.immediateFuture(Collections.singletonList(entryActiveSchedule)); AlarmConditionFilter highTempFilter = new AlarmConditionFilter(); - highTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + highTempFilter.setKey(new AlarmConditionFilterKey(TIME_SERIES, "temperature")); highTempFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate highTemperaturePredicate = new NumericFilterPredicate(); - highTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + highTemperaturePredicate.setOperation(GREATER); highTemperaturePredicate.setValue(new FilterPredicateValue<>( 0.0, null, @@ -1492,10 +1473,10 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { Futures.immediateFuture(Collections.singletonList(entryInactiveSchedule)); AlarmConditionFilter highTempFilter = new AlarmConditionFilter(); - highTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + highTempFilter.setKey(new AlarmConditionFilterKey(TIME_SERIES, "temperature")); highTempFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate highTemperaturePredicate = new NumericFilterPredicate(); - highTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + highTemperaturePredicate.setOperation(GREATER); highTemperaturePredicate.setValue(new FilterPredicateValue<>( 0.0, null, @@ -1593,10 +1574,10 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { Futures.immediateFuture(Optional.of(entry)); AlarmConditionFilter lowTempFilter = new AlarmConditionFilter(); - lowTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + lowTempFilter.setKey(new AlarmConditionFilterKey(TIME_SERIES, "temperature")); lowTempFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate lowTempPredicate = new NumericFilterPredicate(); - lowTempPredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS); + lowTempPredicate.setOperation(LESS); lowTempPredicate.setValue( new FilterPredicateValue<>( 20.0, @@ -1679,10 +1660,10 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { Futures.immediateFuture(Optional.of(entry)); AlarmConditionFilter lowTempFilter = new AlarmConditionFilter(); - lowTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + lowTempFilter.setKey(new AlarmConditionFilterKey(TIME_SERIES, "temperature")); lowTempFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate lowTempPredicate = new NumericFilterPredicate(); - lowTempPredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS); + lowTempPredicate.setOperation(LESS); lowTempPredicate.setValue( new FilterPredicateValue<>( 32.0, @@ -1769,10 +1750,10 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { Futures.immediateFuture(Optional.of(entry)); AlarmConditionFilter lowTempFilter = new AlarmConditionFilter(); - lowTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + lowTempFilter.setKey(new AlarmConditionFilterKey(TIME_SERIES, "temperature")); lowTempFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate lowTempPredicate = new NumericFilterPredicate(); - lowTempPredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + lowTempPredicate.setOperation(GREATER); lowTempPredicate.setValue( new FilterPredicateValue<>( 0.0, @@ -1865,10 +1846,10 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { Futures.immediateFuture(Optional.of(entry)); AlarmConditionFilter lowTempFilter = new AlarmConditionFilter(); - lowTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + lowTempFilter.setKey(new AlarmConditionFilterKey(TIME_SERIES, "temperature")); lowTempFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate lowTempPredicate = new NumericFilterPredicate(); - lowTempPredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + lowTempPredicate.setOperation(GREATER); lowTempPredicate.setValue( new FilterPredicateValue<>( 0.0, @@ -1942,6 +1923,10 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { } private void registerCreateAlarmMock(AlarmApiCallResult a, boolean created) { + registerCreateAlarmMock(a, created, false); + } + + private void registerCreateAlarmMock(AlarmApiCallResult a, boolean created, boolean cleared) { when(a).thenAnswer(invocationOnMock -> { AlarmInfo alarm = new AlarmInfo(new Alarm(new AlarmId(UUID.randomUUID()))); AlarmModificationRequest request = invocationOnMock.getArgument(0); @@ -1950,6 +1935,7 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { .successful(true) .created(created) .modified(true) + .cleared(cleared) .alarm(alarm) .build(); }); @@ -1981,6 +1967,100 @@ public class TbDeviceProfileNodeTest extends AbstractRuleNodeUpgradeTest { } + @Test + public void testAlarmCreateWithAttrAndTsCondition() throws Exception { + init(); + + DeviceProfile deviceProfile = new DeviceProfile(); + DeviceProfileData deviceProfileData = new DeviceProfileData(); + + AlarmConditionFilter filter = getAlarmConditionFilter(TIME_SERIES, "temperature", GREATER, 30.0); + AlarmConditionFilter filter2 = getAlarmConditionFilter(ATTRIBUTE, "battery", LESS, 10.0); + AlarmCondition alarmCondition = new AlarmCondition(); + alarmCondition.setCondition(List.of(filter, filter2)); + AlarmRule createRule = new AlarmRule(); + createRule.setCondition(alarmCondition); + + AlarmConditionFilter filter3 = getAlarmConditionFilter(TIME_SERIES, "temperature", LESS, 10.0); + AlarmConditionFilter filter4 = getAlarmConditionFilter(ATTRIBUTE, "battery", GREATER, 50.0); + AlarmCondition clearCondition = new AlarmCondition(); + clearCondition.setCondition(List.of(filter3, filter4)); + AlarmRule clearRule = new AlarmRule(); + clearRule.setCondition(clearCondition); + + DeviceProfileAlarm dpa = new DeviceProfileAlarm(); + dpa.setId("highTemperatureAlarmID"); + dpa.setAlarmType("highTemperatureAlarm"); + dpa.setCreateRules(new TreeMap<>(Collections.singletonMap(AlarmSeverity.CRITICAL, createRule))); + dpa.setClearRule(clearRule); + + deviceProfileData.setAlarms(Collections.singletonList(dpa)); + deviceProfile.setProfileData(deviceProfileData); + + ListenableFuture> tsKvList = + Futures.immediateFuture(Collections.singletonList(getTsKvEntry("temperature", 35L))); + ListenableFuture> attrList = + Futures.immediateFuture(Collections.emptyList()); + + Mockito.when(cache.get(tenantId, deviceId)).thenReturn(deviceProfile); + Mockito.when(timeseriesService.findLatest(tenantId, deviceId, Collections.singleton("temperature"))) + .thenReturn(tsKvList); + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), any(), anySet())) + .thenReturn(attrList); + Mockito.when(alarmService.findLatestActiveByOriginatorAndType(tenantId, deviceId, "highTemperatureAlarm")).thenReturn(null); + registerCreateAlarmMock(alarmService.createAlarm(any()), true); + + TbMsg theMsg = TbMsg.newMsg() + .type(TbMsgType.ALARM) + .originator(deviceId) + .copyMetaData(TbMsgMetaData.EMPTY) + .data(TbMsg.EMPTY_STRING) + .build(); + when(ctx.newMsg(any(), any(TbMsgType.class), any(), any(), any(), Mockito.anyString())).thenReturn(theMsg); + + // send attribute + ObjectNode data = JacksonUtil.newObjectNode(); + data.put("battery", 8); + TbMsg msg = TbMsg.newMsg() + .type(TbMsgType.POST_ATTRIBUTES_REQUEST) + .originator(deviceId) + .copyMetaData(TbMsgMetaData.EMPTY) + .dataType(TbMsgDataType.JSON) + .data(JacksonUtil.toString(data)) + .build(); + node.onMsg(ctx, msg); + verify(ctx).tellSuccess(msg); + verify(ctx).enqueueForTellNext(theMsg, "Alarm Created"); + verify(ctx, Mockito.never()).tellFailure(Mockito.any(), Mockito.any()); + } + + private TsKvEntry getTsKvEntry(String key, Long value) { + TsKvEntity tsKv = new TsKvEntity(); + tsKv.setKey(10); + tsKv.setLongValue(value); + tsKv.setStrKey(key); + tsKv.setTs(System.currentTimeMillis()); + return tsKv.toData(); + } + + private AlarmCondition getNumericAlarmCondition(AlarmConditionKeyType alarmConditionKeyType, String key, NumericOperation operation, Double value) { + AlarmConditionFilter filter = getAlarmConditionFilter(alarmConditionKeyType, key, operation, value); + AlarmCondition alarmCondition = new AlarmCondition(); + alarmCondition.setCondition(Collections.singletonList(filter)); + return alarmCondition; + } + + private AlarmConditionFilter getAlarmConditionFilter(AlarmConditionKeyType alarmConditionKeyType, String key, NumericOperation operation, Double value) { + AlarmConditionFilter filter = new AlarmConditionFilter(); + filter.setKey(new AlarmConditionFilterKey(alarmConditionKeyType, key)); + filter.setValueType(EntityKeyValueType.NUMERIC); + NumericFilterPredicate highTemperaturePredicate = new NumericFilterPredicate(); + highTemperaturePredicate.setOperation(operation); + highTemperaturePredicate.setValue(new FilterPredicateValue<>(value)); + filter.setPredicate(highTemperaturePredicate); + return filter; + } + @Override protected TbNode getTestNode() { return node; From 3bd2afad8337cde345072ced0e6835267c3335c5 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 11 Apr 2025 09:27:15 +0300 Subject: [PATCH 13/40] Use consumer-properties-per-topic-inline to configure custom topics --- .../src/main/resources/thingsboard.yml | 5 +++ .../server/common/data/TbProperty.java | 8 +++-- .../server/queue/kafka/TbKafkaSettings.java | 36 +++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 716797c2d2..5d5962554d 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1628,6 +1628,11 @@ queue: - key: max.poll.records # Max poll records for edqs.state topic value: "${TB_QUEUE_KAFKA_EDQS_STATE_MAX_POLL_RECORDS:512}" + # If you override any default Kafka topic name using environment variables, you must also specify the related consumer properties + # for the new topic in `consumer-properties-per-topic-inline`. Otherwise, the topic will not inherit its expected configuration (e.g., max.poll.records, timeouts, etc). + # Format: "topic1:key1=value1,key2=value2;topic2:key=value" + # Example: "tb_core_modified.notifications:max.poll.records=10;tb_edge_modified:max.poll.records=10,enable.auto.commit=true" + consumer-properties-per-topic-inline: "${TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE:}" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TbProperty.java b/common/data/src/main/java/org/thingsboard/server/common/data/TbProperty.java index 98fd521ea3..a72bd6d085 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/TbProperty.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TbProperty.java @@ -15,14 +15,16 @@ */ package org.thingsboard.server.common.data; +import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; -/** - * Created by ashvayka on 25.09.18. - */ @Data +@NoArgsConstructor +@AllArgsConstructor public class TbProperty { private String key; private String value; + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index 82b5af179f..e0682f6364 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.kafka; +import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Getter; import lombok.Setter; @@ -36,7 +37,9 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.TbProperty; import org.thingsboard.server.queue.util.PropertyUtils; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -138,6 +141,9 @@ public class TbKafkaSettings { @Value("${queue.kafka.other-inline:}") private String otherInline; + @Value("${queue.kafka.consumer-properties-per-topic-inline:}") + private String consumerPropertiesPerTopicInline; + @Deprecated @Setter private List other; @@ -147,6 +153,14 @@ public class TbKafkaSettings { private volatile AdminClient adminClient; + @PostConstruct + public void initInlineTopicProperties() { + Map> inlineProps = parseTopicPropertyList(consumerPropertiesPerTopicInline); + if (!inlineProps.isEmpty()) { + consumerPropertiesPerTopic.putAll(inlineProps); + } + } + public Properties toConsumerProps(String topic) { Properties props = toProps(); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers); @@ -245,6 +259,28 @@ public class TbKafkaSettings { return props; } + private Map> parseTopicPropertyList(String inlineProperties) { + Map> result = new HashMap<>(); + Map rawTopicToPropertyString = PropertyUtils.getProps(inlineProperties); + + for (Map.Entry entry : rawTopicToPropertyString.entrySet()) { + String topic = entry.getKey().trim(); + String propertiesStr = entry.getValue(); + + List tbProperties = Arrays.stream(propertiesStr.split(",")) + .map(kv -> kv.split("=", 2)) + .filter(kvArr -> kvArr.length == 2) + .map(kvArr -> new TbProperty(kvArr[0].trim(), kvArr[1].trim())) + .toList(); + + if (!tbProperties.isEmpty()) { + result.put(topic, tbProperties); + } + } + + return result; + } + @PreDestroy private void destroy() { if (adminClient != null) { From 82d4cb538112a98ed6425cf56c63e27cceb6731f Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 15 Apr 2025 15:12:18 +0300 Subject: [PATCH 14/40] Add monitoring for calculated fields --- .../monitoring/client/WsClient.java | 69 +++++++++++-------- .../config/transport/TransportInfo.java | 11 ++- .../transport/TransportMonitoringTarget.java | 1 + .../monitoring/data/MonitoredServiceKey.java | 1 + .../data/ServiceFailureException.java | 11 ++- .../monitoring/data/cmd/EntityDataUpdate.java | 18 +++-- .../monitoring/service/BaseHealthChecker.java | 44 ++++++++---- .../service/BaseMonitoringService.java | 37 +++++----- .../service/MonitoringReporter.java | 2 +- .../transport/TransportHealthChecker.java | 44 +++++++++++- .../src/main/resources/tb-monitoring.yml | 11 +++ .../thingsboard/rest/client/RestClient.java | 20 +++++- 12 files changed, 189 insertions(+), 80 deletions(-) diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/client/WsClient.java b/monitoring/src/main/java/org/thingsboard/monitoring/client/WsClient.java index 9246feac27..9bacbdfd45 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/client/WsClient.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/client/WsClient.java @@ -36,8 +36,11 @@ import org.thingsboard.server.common.data.query.EntityListFilter; import javax.net.ssl.SSLParameters; import java.net.URI; import java.nio.channels.NotYetConnectedException; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -48,13 +51,13 @@ import java.util.stream.Collectors; @Slf4j public class WsClient extends WebSocketClient implements AutoCloseable { - public volatile JsonNode lastMsg; + public final List lastMsgs = new ArrayList<>(); private CountDownLatch reply; private CountDownLatch update; private final Lock updateLock = new ReentrantLock(); - private long requestTimeoutMs; + private final long requestTimeoutMs; public WsClient(URI serverUri, long requestTimeoutMs) { super(serverUri); @@ -63,7 +66,6 @@ public class WsClient extends WebSocketClient implements AutoCloseable { @Override public void onOpen(ServerHandshake serverHandshake) { - } @Override @@ -73,8 +75,9 @@ public class WsClient extends WebSocketClient implements AutoCloseable { } updateLock.lock(); try { - lastMsg = JacksonUtil.toJsonNode(s); - log.trace("Received new msg: {}", lastMsg.toPrettyString()); + JsonNode msg = JacksonUtil.toJsonNode(s); + lastMsgs.add(msg); + log.trace("Received new msg: {}", msg.toPrettyString()); if (update != null) { update.countDown(); } @@ -96,11 +99,11 @@ public class WsClient extends WebSocketClient implements AutoCloseable { log.error("WebSocket client error:", e); } - public void registerWaitForUpdate() { + public void registerWaitForUpdates(int count) { updateLock.lock(); try { - lastMsg = null; - update = new CountDownLatch(1); + lastMsgs.clear(); + update = new CountDownLatch(count); } finally { updateLock.unlock(); } @@ -111,6 +114,7 @@ public class WsClient extends WebSocketClient implements AutoCloseable { public void send(String text) throws NotYetConnectedException { updateLock.lock(); try { + lastMsgs.clear(); reply = new CountDownLatch(1); } finally { updateLock.unlock(); @@ -118,19 +122,19 @@ public class WsClient extends WebSocketClient implements AutoCloseable { super.send(text); } - public WsClient subscribeForTelemetry(List devices, String key) { + public WsClient subscribeForTelemetry(List devices, List keys) { EntityDataCmd cmd = new EntityDataCmd(); cmd.setCmdId(RandomUtils.nextInt(0, 1000)); EntityListFilter devicesFilter = new EntityListFilter(); devicesFilter.setEntityType(EntityType.DEVICE); devicesFilter.setEntityList(devices.stream().map(UUID::toString).collect(Collectors.toList())); - EntityDataPageLink pageLink = new EntityDataPageLink(100,0, null, null); + EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, null); EntityDataQuery devicesQuery = new EntityDataQuery(devicesFilter, pageLink, Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); cmd.setQuery(devicesQuery); LatestValueCmd latestCmd = new LatestValueCmd(); - latestCmd.setKeys(List.of(new EntityKey(EntityKeyType.TIME_SERIES, key))); + latestCmd.setKeys(keys.stream().map(key -> new EntityKey(EntityKeyType.TIME_SERIES, key)).toList()); cmd.setLatestCmd(latestCmd); CmdsWrapper wrapper = new CmdsWrapper(); @@ -139,12 +143,12 @@ public class WsClient extends WebSocketClient implements AutoCloseable { return this; } - public JsonNode waitForUpdate(long ms) { + public List waitForUpdates(long ms) { log.trace("update latch count: {}", update.getCount()); try { if (update.await(ms, TimeUnit.MILLISECONDS)) { log.trace("Waited for update"); - return getLastMsg(); + return getLastMsgs(); } } catch (InterruptedException e) { log.debug("Failed to await reply", e); @@ -157,7 +161,8 @@ public class WsClient extends WebSocketClient implements AutoCloseable { try { if (reply.await(requestTimeoutMs, TimeUnit.MILLISECONDS)) { log.trace("Waited for reply"); - return getLastMsg(); + List lastMsgs = getLastMsgs(); + return lastMsgs.isEmpty() ? null : lastMsgs.get(0); } } catch (InterruptedException e) { log.debug("Failed to await reply", e); @@ -166,24 +171,30 @@ public class WsClient extends WebSocketClient implements AutoCloseable { throw new IllegalStateException("No WS reply arrived within " + requestTimeoutMs + " ms"); } - private JsonNode getLastMsg() { - if (lastMsg != null) { - JsonNode errorMsg = lastMsg.get("errorMsg"); - if (errorMsg != null && !errorMsg.isNull() && StringUtils.isNotEmpty(errorMsg.asText())) { - throw new RuntimeException("WS error from server: " + errorMsg.asText()); - } else { - return lastMsg; - } - } else { - return null; + private List getLastMsgs() { + if (lastMsgs.isEmpty()) { + return lastMsgs; + } + List errors = lastMsgs.stream() + .map(msg -> msg.get("errorMsg")) + .filter(errorMsg -> errorMsg != null && !errorMsg.isNull() && StringUtils.isNotEmpty(errorMsg.asText())) + .toList(); + if (!errors.isEmpty()) { + throw new RuntimeException("WS error from server: " + errors.stream() + .map(JsonNode::asText) + .collect(Collectors.joining(", "))); } + return lastMsgs; } - public Object getTelemetryUpdate(UUID deviceId, String key) { - JsonNode lastMsg = getLastMsg(); - if (lastMsg == null || lastMsg.isNull()) return null; - EntityDataUpdate update = JacksonUtil.treeToValue(lastMsg, EntityDataUpdate.class); - return update.getLatest(deviceId, key); + public Map getLatest(UUID deviceId) { + Map updates = new HashMap<>(); + getLastMsgs().forEach(msg -> { + EntityDataUpdate update = JacksonUtil.treeToValue(msg, EntityDataUpdate.class); + Map latest = update.getLatest(deviceId); + updates.putAll(latest); + }); + return updates; } @Override diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportInfo.java b/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportInfo.java index 208d28ffee..6b77e1e268 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportInfo.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportInfo.java @@ -20,16 +20,15 @@ import lombok.Data; @Data public class TransportInfo { - private final TransportType transportType; - private final String baseUrl; - private final String queue; + private final TransportType type; + private final TransportMonitoringTarget target; @Override public String toString() { - if (queue.equals("Main")) { - return String.format("*%s* (%s)", transportType.getName(), baseUrl); + if (target.getQueue().equals("Main")) { + return String.format("*%s* (%s)", type.getName(), target.getBaseUrl()); } else { - return String.format("*%s* (%s) _%s_", transportType.getName(), baseUrl, queue); + return String.format("*%s* (%s) _%s_", type.getName(), target.getBaseUrl(), target.getQueue()); } } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringTarget.java b/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringTarget.java index 1e6a3c8509..e8a9ab03fa 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringTarget.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringTarget.java @@ -28,6 +28,7 @@ public class TransportMonitoringTarget implements MonitoringTarget { private DeviceConfig device; // set manually during initialization private String queue; private boolean checkDomainIps; + private String namePrefix; @Override public UUID getDeviceId() { diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/data/MonitoredServiceKey.java b/monitoring/src/main/java/org/thingsboard/monitoring/data/MonitoredServiceKey.java index 9c3ee5b786..7579d75231 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/data/MonitoredServiceKey.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/data/MonitoredServiceKey.java @@ -19,5 +19,6 @@ public class MonitoredServiceKey { public static final String GENERAL = "Monitoring"; public static final String EDQS = "*EDQS*"; + public static final String CF = "*CF*"; } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/data/ServiceFailureException.java b/monitoring/src/main/java/org/thingsboard/monitoring/data/ServiceFailureException.java index 5f46514bd1..b2592b8719 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/data/ServiceFailureException.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/data/ServiceFailureException.java @@ -15,14 +15,21 @@ */ package org.thingsboard.monitoring.data; +import lombok.Getter; + +@Getter public class ServiceFailureException extends RuntimeException { - public ServiceFailureException(Throwable cause) { + private final Object serviceKey; + + public ServiceFailureException(Object serviceKey, Throwable cause) { super(cause.getMessage(), cause); + this.serviceKey = serviceKey; } - public ServiceFailureException(String message) { + public ServiceFailureException(Object serviceKey, String message) { super(message); + this.serviceKey = serviceKey; } } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/data/cmd/EntityDataUpdate.java b/monitoring/src/main/java/org/thingsboard/monitoring/data/cmd/EntityDataUpdate.java index b82dcbf54d..0706dedcf3 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/data/cmd/EntityDataUpdate.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/data/cmd/EntityDataUpdate.java @@ -19,9 +19,11 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.common.data.query.TsValue; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; @Data @@ -31,14 +33,16 @@ public class EntityDataUpdate { @JsonIgnoreProperties(ignoreUnknown = true) private List update; - public String getLatest(UUID entityId, String key) { - if (update == null) return null; - - return update.stream() + public Map getLatest(UUID entityId) { + if (update == null || update.isEmpty()) { + return Collections.emptyMap(); + } + Map result = new HashMap<>(); + update.stream() .filter(entityData -> entityData.getEntityId().getId().equals(entityId)).findFirst() .map(EntityData::getLatest).map(latest -> latest.get(EntityKeyType.TIME_SERIES)) - .map(latest -> latest.get(key)).map(TsValue::getValue) - .orElse(null); + .ifPresent(latest -> latest.forEach((key, tsValue) -> result.put(key, tsValue.getValue()))); + return result; } } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java index 482b3a30fb..34e0ebc4c5 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java @@ -52,11 +52,14 @@ public abstract class BaseHealthChecker> associates = new HashMap<>(); public static final String TEST_TELEMETRY_KEY = "testData"; + public static final String TEST_CF_TELEMETRY_KEY = "testDataCf"; @PostConstruct private void init() { @@ -68,7 +71,8 @@ public abstract class BaseHealthChecker latest = wsClient.getLatest(target.getDeviceId()); + if (latest.isEmpty()) { + throw new ServiceFailureException(info, "No WS update arrived within " + resultCheckTimeoutMs + " ms"); + } + String actualValue = latest.get(TEST_TELEMETRY_KEY); + if (!testValue.equals(actualValue)) { + throw new ServiceFailureException(info, "Was expecting value " + testValue + " but got " + actualValue); + } + if (checkCalculatedFields) { + String cfTestValue = testValue + "-cf"; + String actualCfValue = latest.get(TEST_CF_TELEMETRY_KEY); + if (actualCfValue == null) { + throw new ServiceFailureException(MonitoredServiceKey.CF, "No CF value arrived"); + } else if (!cfTestValue.equals(actualCfValue)) { + throw new ServiceFailureException(MonitoredServiceKey.CF, "Was expecting CF value " + cfTestValue + " but got " + actualCfValue); + } else { + reporter.serviceIsOk(MonitoredServiceKey.CF); + } } reporter.reportLatency(Latencies.wsUpdate(getKey()), stopWatch.getTime()); } @@ -121,6 +138,7 @@ public abstract class BaseHealthChecker, T extends MonitoringTarget> { @@ -79,7 +81,9 @@ public abstract class BaseMonitoringService, T ext protected ApplicationContext applicationContext; @Value("${monitoring.edqs.enabled:false}") - private boolean edqsMonitoringEnabled; + private boolean checkEdqs; + @Value("${monitoring.calculated_fields.enabled:true}") + protected boolean checkCalculatedFields; @PostConstruct private void init() { @@ -121,7 +125,7 @@ public abstract class BaseMonitoringService, T ext try (WsClient wsClient = wsClientFactory.createClient(accessToken)) { stopWatch.start(); - wsClient.subscribeForTelemetry(devices, TransportHealthChecker.TEST_TELEMETRY_KEY).waitForReply(); + wsClient.subscribeForTelemetry(devices, getTestTelemetryKeys()).waitForReply(); reporter.reportLatency(Latencies.WS_SUBSCRIBE, stopWatch.getTime()); for (BaseHealthChecker healthChecker : healthCheckers) { @@ -129,22 +133,17 @@ public abstract class BaseMonitoringService, T ext } } - if (edqsMonitoringEnabled) { - try { - stopWatch.start(); - checkEdqs(); - reporter.reportLatency(Latencies.EDQS_QUERY, stopWatch.getTime()); - - reporter.serviceIsOk(MonitoredServiceKey.EDQS); - } catch (ServiceFailureException e) { - reporter.serviceFailure(MonitoredServiceKey.EDQS, e); - } catch (Exception e) { - reporter.serviceFailure(MonitoredServiceKey.GENERAL, e); - } + if (checkEdqs) { + stopWatch.start(); + checkEdqs(); + reporter.reportLatency(Latencies.EDQS_QUERY, stopWatch.getTime()); + reporter.serviceIsOk(MonitoredServiceKey.EDQS); } reporter.reportLatencies(tbClient); log.debug("Finished {}", getName()); + } catch (ServiceFailureException e) { + reporter.serviceFailure(e.getServiceKey(), e); } catch (Throwable error) { try { reporter.serviceFailure(MonitoredServiceKey.GENERAL, error); @@ -199,7 +198,7 @@ public abstract class BaseMonitoringService, T ext .collect(Collectors.toSet()); Set missing = Sets.difference(new HashSet<>(this.devices), devices); if (!missing.isEmpty()) { - throw new ServiceFailureException("Missing devices in the response: " + missing); + throw new ServiceFailureException(MonitoredServiceKey.EDQS, "Missing devices in the response: " + missing); } result.getData().stream() @@ -211,7 +210,7 @@ public abstract class BaseMonitoringService, T ext Stream.of("name", "type", "testData").forEach(key -> { TsValue value = values.get(key); if (value == null || StringUtils.isBlank(value.getValue())) { - throw new ServiceFailureException("Missing " + key + " for device " + entityData.getEntityId()); + throw new ServiceFailureException(MonitoredServiceKey.EDQS, "Missing " + key + " for device " + entityData.getEntityId()); } }); }); @@ -232,6 +231,10 @@ public abstract class BaseMonitoringService, T ext .collect(Collectors.toSet()); } + private List getTestTelemetryKeys() { + return checkCalculatedFields ? List.of(TEST_TELEMETRY_KEY, TEST_CF_TELEMETRY_KEY) : List.of(TEST_TELEMETRY_KEY); + } + private void stopHealthChecker(BaseHealthChecker healthChecker) throws Exception { healthChecker.destroyClient(); devices.remove(healthChecker.getTarget().getDeviceId()); diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringReporter.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringReporter.java index 3554731e58..62ed0d74aa 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringReporter.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringReporter.java @@ -113,7 +113,7 @@ public class MonitoringReporter { public void serviceFailure(Object serviceKey, Throwable error) { if (log.isDebugEnabled()) { - log.error("Error occurred", error); + log.error("[{}] Error occurred", serviceKey, error); } int failuresCount = failuresCounters.computeIfAbsent(serviceKey, k -> new AtomicInteger()).incrementAndGet(); ServiceFailureNotification notification = new ServiceFailureNotification(serviceKey, error, failuresCount); diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java index f5e6cb5469..b892f5d609 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java @@ -32,6 +32,14 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MBootstrapClientCredentials; import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MDeviceCredentials; import org.thingsboard.server.common.data.device.credentials.lwm2m.NoSecBootstrapClientCredential; @@ -47,6 +55,8 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; +import java.util.Map; + @Slf4j public abstract class TransportHealthChecker extends BaseHealthChecker { @@ -74,7 +84,7 @@ public abstract class TransportHealthChecker getImages(PageLink pageLink, boolean includeSystemImages) { - return this.getImages(pageLink, null, includeSystemImages); + return this.getImages(pageLink, null, includeSystemImages); } public PageData getImages(PageLink pageLink, ResourceSubType imageSubType, boolean includeSystemImages) { @@ -4056,6 +4057,21 @@ public class RestClient implements Closeable { timeout).getBody(); } + public CalculatedField saveCalculatedField(CalculatedField calculatedField) { + return restTemplate.postForEntity(baseURL + "/api/calculatedField", calculatedField, CalculatedField.class).getBody(); + } + + public PageData getCalculatedFieldsByEntityId(EntityId entityId, PageLink pageLink) { + Map params = new HashMap<>(); + addPageLinkToParam(params, pageLink); + return restTemplate.exchange( + baseURL + "/api/" + entityId.getEntityType() + "/" + entityId.getId() + "/calculatedFields?" + getUrlParams(pageLink), + HttpMethod.GET, HttpEntity.EMPTY, + new ParameterizedTypeReference>() { + }, params).getBody(); + + } + private String getTimeUrlParams(TimePageLink pageLink) { String urlParams = getUrlParams(pageLink); if (pageLink.getStartTime() != null) { From a8a0083bb2d430288794badf90501436b9a8cdaf Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 15 Apr 2025 15:16:55 +0300 Subject: [PATCH 15/40] Monitoring: trim device name --- .../service/transport/TransportHealthChecker.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java index b892f5d609..d002546335 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java @@ -97,7 +97,7 @@ public abstract class TransportHealthChecker Date: Tue, 15 Apr 2025 15:37:20 +0300 Subject: [PATCH 16/40] Monitoring: minor refactoring --- .../monitoring/service/BaseHealthChecker.java | 8 ++++---- .../service/transport/TransportHealthChecker.java | 11 ++++++++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java index 34e0ebc4c5..1e9cdbe191 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java @@ -52,8 +52,6 @@ public abstract class BaseHealthChecker> associates = new HashMap<>(); @@ -71,7 +69,7 @@ public abstract class BaseHealthChecker extends BaseHealthChecker { + @Value("${monitoring.calculated_fields.enabled:true}") + private boolean calculatedFieldsMonitoringEnabled; + public TransportHealthChecker(C config, TransportMonitoringTarget target) { super(config, target); } @@ -100,7 +104,7 @@ public abstract class TransportHealthChecker Date: Tue, 22 Apr 2025 11:23:42 +0300 Subject: [PATCH 17/40] Cleanup deprecated API in SubscriptionManagerService --- .../queue/DefaultTbCoreConsumerService.java | 17 ++++---------- .../DefaultSubscriptionManagerService.java | 22 +++++++------------ .../SubscriptionManagerService.java | 9 -------- 3 files changed, 12 insertions(+), 36 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 47a0c473e9..d2fe8a837d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -552,19 +552,10 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService keys, TbCallback callback) { - onAttributesDelete(tenantId, entityId, scope, keys, false, callback); - } - - @Override - public void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List keys, boolean notifyDevice, TbCallback callback) { - processAttributesUpdate(entityId, scope, - keys.stream().map(key -> new BaseAttributeKvEntry(0, new StringDataEntry(key, ""))).collect(Collectors.toList())); - if (entityId.getEntityType() == EntityType.DEVICE && TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope) && notifyDevice) { - clusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onDelete(tenantId, new DeviceId(entityId.getId()), scope, keys), null); + try { + List deletedEntries = keys.stream() + .map(key -> new BaseAttributeKvEntry(0L, new StringDataEntry(key, ""))) + .toList(); + processAttributesUpdate(entityId, scope, deletedEntries); + } catch (Exception e) { + callback.onFailure(e); + return; } callback.onSuccess(); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java index 57d4fda5f8..3fd58a1243 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java @@ -41,15 +41,6 @@ public interface SubscriptionManagerService extends ApplicationListener keys, TbCallback empty); - /** - * This method is retained solely for backwards compatibility, specifically to handle - * legacy proto messages that include the notifyDevice field. - * - * @deprecated as of 4.0, this method will be removed in future releases. - */ - @Deprecated(forRemoval = true, since = "4.0") - void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List keys, boolean notifyDevice, TbCallback empty); - void onTimeSeriesDelete(TenantId tenantId, EntityId entityId, List keys, TbCallback callback); void onAlarmUpdate(TenantId tenantId, EntityId entityId, AlarmInfo alarm, TbCallback callback); From 49289da5a8e15e99d4a8776f935a2f38a2d6f1fc Mon Sep 17 00:00:00 2001 From: Tarnavskiy Date: Wed, 23 Apr 2025 17:55:51 +0300 Subject: [PATCH 18/40] Fixed an issue when the user could break the pagination settings validation by switching between the basic/advanced mode tabs in table-widgets --- .../widget/lib/alarm/alarms-table-widget.component.ts | 8 +++++--- .../widget/lib/entity/entities-table-widget.component.ts | 8 +++++--- .../widget/lib/rpc/persistent-table.component.ts | 8 +++++--- .../home/components/widget/lib/table-widget.models.ts | 6 +++--- .../widget/lib/timeseries-table-widget.component.ts | 8 +++++--- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts index 7384f76b62..31843506cc 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts @@ -392,10 +392,12 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, this.rowStylesInfo = getRowStyleInfo(this.ctx, this.settings, 'alarm, ctx'); const pageSize = this.settings.defaultPageSize; - let pageStepIncrement = this.settings.pageStepIncrement; - let pageStepCount = this.settings.pageStepCount; + let pageStepIncrement = Number.isInteger(this.settings.pageStepIncrement) && this.settings.pageStepIncrement > 0 ? + this.settings.pageStepIncrement : null; + let pageStepCount = Number.isInteger(this.settings.pageStepCount) && this.settings.pageStepCount > 0 + && this.settings.pageStepCount <= 100 ? this.settings.pageStepCount : null; - if (isDefined(pageSize) && isNumber(pageSize) && pageSize > 0) { + if (Number.isInteger(pageSize) && pageSize > 0) { this.defaultPageSize = pageSize; } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.ts index 653c388364..a80c6896bb 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.ts @@ -311,10 +311,12 @@ export class EntitiesTableWidgetComponent extends PageComponent implements OnIni this.rowStylesInfo = getRowStyleInfo(this.ctx, this.settings, 'entity, ctx'); const pageSize = this.settings.defaultPageSize; - let pageStepIncrement = this.settings.pageStepIncrement; - let pageStepCount = this.settings.pageStepCount; + let pageStepIncrement = Number.isInteger(this.settings.pageStepIncrement) && this.settings.pageStepIncrement > 0 ? + this.settings.pageStepIncrement : null; + let pageStepCount = Number.isInteger(this.settings.pageStepCount) && this.settings.pageStepCount > 0 + && this.settings.pageStepCount <= 100 ? this.settings.pageStepCount : null; - if (isDefined(pageSize) && isNumber(pageSize) && pageSize > 0) { + if (Number.isInteger(pageSize) && pageSize > 0) { this.defaultPageSize = pageSize; } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts index 586f425b13..0647f41771 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts @@ -207,10 +207,12 @@ export class PersistentTableComponent extends PageComponent implements OnInit, O this.displayedColumns = [...this.displayTableColumns]; const pageSize = this.settings.defaultPageSize; - let pageStepIncrement = this.settings.pageStepIncrement; - let pageStepCount = this.settings.pageStepCount; + let pageStepIncrement = Number.isInteger(this.settings.pageStepIncrement) && this.settings.pageStepIncrement > 0 ? + this.settings.pageStepIncrement : null; + let pageStepCount = Number.isInteger(this.settings.pageStepCount) && this.settings.pageStepCount > 0 + && this.settings.pageStepCount <= 100 ? this.settings.pageStepCount : null; - if (isDefined(pageSize) && isNumber(pageSize) && pageSize > 0) { + if (Number.isInteger(pageSize) && pageSize > 0) { this.defaultPageSize = pageSize; } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts index 8aadc62ee4..eb944e3c65 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts @@ -16,7 +16,7 @@ import { EntityId } from '@shared/models/id/entity-id'; import { DataKey, FormattedData, WidgetActionDescriptor, WidgetConfig } from '@shared/models/widget.models'; -import { getDescendantProp, isDefined, isDefinedAndNotNull, isNotEmptyStr } from '@core/utils'; +import { getDescendantProp, isDefined, isNotEmptyStr } from '@core/utils'; import { AlarmDataInfo, alarmFields } from '@shared/models/alarm.models'; import tinycolor from 'tinycolor2'; import { Direction } from '@shared/models/page/sort-order'; @@ -564,8 +564,8 @@ export function getHeaderTitle(dataKey: DataKey, keySettings: TableWidgetDataKey export function buildPageStepSizeValues(pageStepCount: number, pageStepIncrement: number): Array { const pageSteps: Array = []; - if (isDefinedAndNotNull(pageStepCount) && pageStepCount > 0 && pageStepCount <= 100 && - isDefinedAndNotNull(pageStepIncrement) && pageStepIncrement > 0) { + if (Number.isInteger(pageStepCount) && pageStepCount > 0 && pageStepCount <= 100 && + Number.isInteger(pageStepIncrement) && pageStepIncrement > 0) { for (let i = 1; i <= pageStepCount; i++) { pageSteps.push(pageStepIncrement * i); } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts index 2653b79059..010e49fe32 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts @@ -352,10 +352,12 @@ export class TimeseriesTableWidgetComponent extends PageComponent implements OnI this.rowStylesInfo = getRowStyleInfo(this.ctx, this.settings, 'rowData, ctx'); const pageSize = this.settings.defaultPageSize; - let pageStepIncrement = this.settings.pageStepIncrement; - let pageStepCount = this.settings.pageStepCount; + let pageStepIncrement = Number.isInteger(this.settings.pageStepIncrement) && this.settings.pageStepIncrement > 0 ? + this.settings.pageStepIncrement : null; + let pageStepCount = Number.isInteger(this.settings.pageStepCount) && this.settings.pageStepCount > 0 + && this.settings.pageStepCount <= 100 ? this.settings.pageStepCount : null; - if (isDefined(pageSize) && isNumber(pageSize) && pageSize > 0) { + if (Number.isInteger(pageSize) && pageSize > 0) { this.defaultPageSize = pageSize; } From 358e635805dfde918eedc4543f839d8e3115a9ce Mon Sep 17 00:00:00 2001 From: Tarnavskiy Date: Wed, 23 Apr 2025 18:34:58 +0300 Subject: [PATCH 19/40] Code optimization for a fix of an issue with broken pagination settings validation in table-widgets --- .../widget/lib/alarm/alarms-table-widget.component.ts | 9 ++++----- .../lib/entity/entities-table-widget.component.ts | 10 +++++----- .../widget/lib/rpc/persistent-table.component.ts | 10 +++++----- .../home/components/widget/lib/table-widget.models.ts | 11 +++++++++-- .../widget/lib/timeseries-table-widget.component.ts | 9 ++++----- 5 files changed, 27 insertions(+), 22 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts index 31843506cc..5321e27c05 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts @@ -43,7 +43,6 @@ import { isDefined, isDefinedAndNotNull, isNotEmptyStr, - isNumber, isObject, isUndefined } from '@core/utils'; @@ -77,6 +76,8 @@ import { getHeaderTitle, getRowStyleInfo, getTableCellButtonActions, + isValidPageStepCount, + isValidPageStepIncrement, noDataMessage, prepareTableCellButtonActions, RowStyleInfo, @@ -392,10 +393,8 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, this.rowStylesInfo = getRowStyleInfo(this.ctx, this.settings, 'alarm, ctx'); const pageSize = this.settings.defaultPageSize; - let pageStepIncrement = Number.isInteger(this.settings.pageStepIncrement) && this.settings.pageStepIncrement > 0 ? - this.settings.pageStepIncrement : null; - let pageStepCount = Number.isInteger(this.settings.pageStepCount) && this.settings.pageStepCount > 0 - && this.settings.pageStepCount <= 100 ? this.settings.pageStepCount : null; + let pageStepIncrement = isValidPageStepIncrement(this.settings.pageStepIncrement) ? this.settings.pageStepIncrement : null; + let pageStepCount = isValidPageStepCount(this.settings.pageStepCount) ? this.settings.pageStepCount : null; if (Number.isInteger(pageSize) && pageSize > 0) { this.defaultPageSize = pageSize; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.ts index a80c6896bb..2a87af2519 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.ts @@ -42,7 +42,7 @@ import { import { IWidgetSubscription } from '@core/api/widget-api.models'; import { UtilsService } from '@core/services/utils.service'; import { TranslateService } from '@ngx-translate/core'; -import { deepClone, hashCode, isDefined, isDefinedAndNotNull, isNumber, isObject, isUndefined } from '@core/utils'; +import { deepClone, hashCode, isDefined, isDefinedAndNotNull, isObject, isUndefined } from '@core/utils'; import cssjs from '@core/css/css'; import { CollectionViewer, DataSource } from '@angular/cdk/collections'; import { DataKeyType } from '@shared/models/telemetry/telemetry.models'; @@ -75,6 +75,8 @@ import { getHeaderTitle, getRowStyleInfo, getTableCellButtonActions, + isValidPageStepCount, + isValidPageStepIncrement, noDataMessage, prepareTableCellButtonActions, RowStyleInfo, @@ -311,10 +313,8 @@ export class EntitiesTableWidgetComponent extends PageComponent implements OnIni this.rowStylesInfo = getRowStyleInfo(this.ctx, this.settings, 'entity, ctx'); const pageSize = this.settings.defaultPageSize; - let pageStepIncrement = Number.isInteger(this.settings.pageStepIncrement) && this.settings.pageStepIncrement > 0 ? - this.settings.pageStepIncrement : null; - let pageStepCount = Number.isInteger(this.settings.pageStepCount) && this.settings.pageStepCount > 0 - && this.settings.pageStepCount <= 100 ? this.settings.pageStepCount : null; + let pageStepIncrement = isValidPageStepIncrement(this.settings.pageStepIncrement) ? this.settings.pageStepIncrement : null; + let pageStepCount = isValidPageStepCount(this.settings.pageStepCount) ? this.settings.pageStepCount : null; if (Number.isInteger(pageSize) && pageSize > 0) { this.defaultPageSize = pageSize; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts index 0647f41771..60f522b192 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts @@ -36,6 +36,8 @@ import { BehaviorSubject, merge, Observable, of, ReplaySubject, Subject, throwEr import { catchError, map, tap } from 'rxjs/operators'; import { constructTableCssString, + isValidPageStepCount, + isValidPageStepIncrement, noDataMessage, TableCellButtonActionDescriptor, TableWidgetSettings @@ -43,7 +45,7 @@ import { import cssjs from '@core/css/css'; import { UtilsService } from '@core/services/utils.service'; import { TranslateService } from '@ngx-translate/core'; -import { hashCode, isDefined, isDefinedAndNotNull, isNumber, parseHttpErrorMessage } from '@core/utils'; +import { hashCode, isDefined, isDefinedAndNotNull, parseHttpErrorMessage } from '@core/utils'; import { CollectionViewer, DataSource } from '@angular/cdk/collections'; import { emptyPageData, PageData } from '@shared/models/page/page-data'; import { @@ -207,10 +209,8 @@ export class PersistentTableComponent extends PageComponent implements OnInit, O this.displayedColumns = [...this.displayTableColumns]; const pageSize = this.settings.defaultPageSize; - let pageStepIncrement = Number.isInteger(this.settings.pageStepIncrement) && this.settings.pageStepIncrement > 0 ? - this.settings.pageStepIncrement : null; - let pageStepCount = Number.isInteger(this.settings.pageStepCount) && this.settings.pageStepCount > 0 - && this.settings.pageStepCount <= 100 ? this.settings.pageStepCount : null; + let pageStepIncrement = isValidPageStepIncrement(this.settings.pageStepIncrement) ? this.settings.pageStepIncrement : null; + let pageStepCount = isValidPageStepCount(this.settings.pageStepCount) ? this.settings.pageStepCount : null; if (Number.isInteger(pageSize) && pageSize > 0) { this.defaultPageSize = pageSize; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts index eb944e3c65..1ff7a21d71 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts @@ -564,11 +564,18 @@ export function getHeaderTitle(dataKey: DataKey, keySettings: TableWidgetDataKey export function buildPageStepSizeValues(pageStepCount: number, pageStepIncrement: number): Array { const pageSteps: Array = []; - if (Number.isInteger(pageStepCount) && pageStepCount > 0 && pageStepCount <= 100 && - Number.isInteger(pageStepIncrement) && pageStepIncrement > 0) { + if (isValidPageStepCount(pageStepCount) && isValidPageStepIncrement(pageStepIncrement)) { for (let i = 1; i <= pageStepCount; i++) { pageSteps.push(pageStepIncrement * i); } } return pageSteps; } + +export function isValidPageStepIncrement(value: number): boolean { + return Number.isInteger(value) && value > 0; +} + +export function isValidPageStepCount(value: number): boolean { + return Number.isInteger(value) && value > 0 && value <= 100; +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts index 010e49fe32..2e3a64c190 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts @@ -49,7 +49,6 @@ import { hashCode, isDefined, isDefinedAndNotNull, - isNumber, isObject, isUndefined } from '@core/utils'; @@ -85,6 +84,8 @@ import { getColumnSelectionAvailability, getRowStyleInfo, getTableCellButtonActions, + isValidPageStepCount, + isValidPageStepIncrement, noDataMessage, prepareTableCellButtonActions, RowStyleInfo, @@ -352,10 +353,8 @@ export class TimeseriesTableWidgetComponent extends PageComponent implements OnI this.rowStylesInfo = getRowStyleInfo(this.ctx, this.settings, 'rowData, ctx'); const pageSize = this.settings.defaultPageSize; - let pageStepIncrement = Number.isInteger(this.settings.pageStepIncrement) && this.settings.pageStepIncrement > 0 ? - this.settings.pageStepIncrement : null; - let pageStepCount = Number.isInteger(this.settings.pageStepCount) && this.settings.pageStepCount > 0 - && this.settings.pageStepCount <= 100 ? this.settings.pageStepCount : null; + let pageStepIncrement = isValidPageStepIncrement(this.settings.pageStepIncrement) ? this.settings.pageStepIncrement : null; + let pageStepCount = isValidPageStepCount(this.settings.pageStepCount) ? this.settings.pageStepCount : null; if (Number.isInteger(pageSize) && pageSize > 0) { this.defaultPageSize = pageSize; From edd7d6392a4fe740f074520a478a52dfe150f408 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Thu, 24 Apr 2025 16:52:31 +0300 Subject: [PATCH 20/40] Improvement after review --- .../src/main/resources/thingsboard.yml | 7 ++-- .../server/queue/kafka/TbKafkaSettings.java | 32 +++++++++---------- .../server/queue/util/PropertyUtils.java | 17 ++++++++++ .../queue/kafka/TbKafkaSettingsTest.java | 20 +++++++++++- 4 files changed, 55 insertions(+), 21 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index ebd001aa97..e013473f98 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1630,9 +1630,10 @@ queue: value: "${TB_QUEUE_KAFKA_EDQS_STATE_MAX_POLL_RECORDS:512}" # If you override any default Kafka topic name using environment variables, you must also specify the related consumer properties # for the new topic in `consumer-properties-per-topic-inline`. Otherwise, the topic will not inherit its expected configuration (e.g., max.poll.records, timeouts, etc). - # Format: "topic1:key1=value1,key2=value2;topic2:key=value" - # Example: "tb_core_modified.notifications:max.poll.records=10;tb_edge_modified:max.poll.records=10,enable.auto.commit=true" - consumer-properties-per-topic-inline: "${TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE:}" + # Each entry sets a single property for a specific topic. To define multiple properties for a topic, repeat the topic key. + # Format: "topic1:key=value;topic1:key=value;topic2:key=value" + # Example: tb_core_updated:max.poll.records=10;tb_core_updated:bootstrap.servers=kafka1:9092,kafka2:9092;tb_edge_updated:auto.offset.reset=latest + consumer-properties-per-topic-inline: "${TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE:tb_core_updated:max.poll.records=10;tb_core_updated:enable.auto.commit=true;tb_core_updated:bootstrap.servers=kafka1:9092,kafka2:9092;tb_edge_updated:max.poll.records=5;tb_edge_updated:auto.offset.reset=latest}" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index e0682f6364..06ccfe3a69 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -37,9 +37,8 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.TbProperty; import org.thingsboard.server.queue.util.PropertyUtils; -import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -149,7 +148,7 @@ public class TbKafkaSettings { private List other; @Setter - private Map> consumerPropertiesPerTopic = Collections.emptyMap(); + private Map> consumerPropertiesPerTopic = new HashMap<>(); private volatile AdminClient adminClient; @@ -260,23 +259,22 @@ public class TbKafkaSettings { } private Map> parseTopicPropertyList(String inlineProperties) { + Map> grouped = PropertyUtils.getGroupedProps(inlineProperties); Map> result = new HashMap<>(); - Map rawTopicToPropertyString = PropertyUtils.getProps(inlineProperties); - for (Map.Entry entry : rawTopicToPropertyString.entrySet()) { - String topic = entry.getKey().trim(); - String propertiesStr = entry.getValue(); - - List tbProperties = Arrays.stream(propertiesStr.split(",")) - .map(kv -> kv.split("=", 2)) - .filter(kvArr -> kvArr.length == 2) - .map(kvArr -> new TbProperty(kvArr[0].trim(), kvArr[1].trim())) - .toList(); - - if (!tbProperties.isEmpty()) { - result.put(topic, tbProperties); + grouped.forEach((topic, entries) -> { + Map merged = new LinkedHashMap<>(); + for (String entry : entries) { + String[] kv = entry.split("=", 2); + if (kv.length == 2) { + merged.put(kv[0].trim(), kv[1].trim()); + } } - } + List props = merged.entrySet().stream() + .map(e -> new TbProperty(e.getKey(), e.getValue())) + .toList(); + result.put(topic, props); + }); return result; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java index 6030eb278d..629ee29f6f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java @@ -17,7 +17,9 @@ package org.thingsboard.server.queue.util; import org.thingsboard.server.common.data.StringUtils; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.Function; @@ -38,6 +40,21 @@ public class PropertyUtils { return configs; } + public static Map> getGroupedProps(String properties) { + Map> configs = new HashMap<>(); + if (StringUtils.isNotEmpty(properties)) { + for (String property : properties.split(";")) { + if (StringUtils.isNotEmpty(property)) { + int delimiterPosition = property.indexOf(":"); + String topic = property.substring(0, delimiterPosition).trim(); + String value = property.substring(delimiterPosition + 1).trim(); + configs.computeIfAbsent(topic, k -> new ArrayList<>()).add(value); + } + } + } + return configs; + } + public static Map getProps(Map defaultProperties, String propertiesStr) { return getProps(defaultProperties, propertiesStr, PropertyUtils::getProps); } diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java index 5cdebc3996..ad026c63aa 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java @@ -33,6 +33,12 @@ import static org.mockito.Mockito.spy; "queue.type=kafka", "queue.kafka.bootstrap.servers=localhost:9092", "queue.kafka.other-inline=metrics.recording.level:INFO;metrics.sample.window.ms:30000", + "queue.kafka.consumer-properties-per-topic-inline=" + + "tb_core_updated:max.poll.records=10;" + + "tb_core_updated:enable.auto.commit=true;" + + "tb_core_updated:bootstrap.servers=kafka1:9092,kafka2:9092;" + + "tb_edge_updated:max.poll.records=5;" + + "tb_edge_updated:auto.offset.reset=latest" }) class TbKafkaSettingsTest { @@ -79,4 +85,16 @@ class TbKafkaSettingsTest { Mockito.verify(settings).configureSSL(any()); } -} \ No newline at end of file + @Test + void givenMultipleTopicsInInlineConfig_whenParsed_thenEachTopicGetsExpectedProperties() { + Properties coreProps = settings.toConsumerProps("tb_core_updated"); + assertThat(coreProps.getProperty("max.poll.records")).isEqualTo("10"); + assertThat(coreProps.getProperty("enable.auto.commit")).isEqualTo("true"); + assertThat(coreProps.getProperty("bootstrap.servers")).isEqualTo("kafka1:9092,kafka2:9092"); + + Properties edgeProps = settings.toConsumerProps("tb_edge_updated"); + assertThat(edgeProps.getProperty("max.poll.records")).isEqualTo("5"); + assertThat(edgeProps.getProperty("auto.offset.reset")).isEqualTo("latest"); + } + +} From b516583c6dbcf75c0cc0afcb390d7316fdf8576b Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Thu, 24 Apr 2025 16:53:57 +0300 Subject: [PATCH 21/40] Remove env setup config --- application/src/main/resources/thingsboard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index e013473f98..9710f38a3a 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1633,7 +1633,7 @@ queue: # Each entry sets a single property for a specific topic. To define multiple properties for a topic, repeat the topic key. # Format: "topic1:key=value;topic1:key=value;topic2:key=value" # Example: tb_core_updated:max.poll.records=10;tb_core_updated:bootstrap.servers=kafka1:9092,kafka2:9092;tb_edge_updated:auto.offset.reset=latest - consumer-properties-per-topic-inline: "${TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE:tb_core_updated:max.poll.records=10;tb_core_updated:enable.auto.commit=true;tb_core_updated:bootstrap.servers=kafka1:9092,kafka2:9092;tb_edge_updated:max.poll.records=5;tb_edge_updated:auto.offset.reset=latest}" + consumer-properties-per-topic-inline: "${TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE:}" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms From 876237fede7d12ac1fc9c6cf538576f086a239a5 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Fri, 25 Apr 2025 14:40:16 +0300 Subject: [PATCH 22/40] Send the `ALARM_DELETE` event only after the alarm is successfully deleted --- .../server/controller/AlarmController.java | 2 +- .../entitiy/AbstractTbEntityService.java | 14 -- .../entitiy/alarm/DefaultTbAlarmService.java | 23 ++- .../service/entitiy/alarm/TbAlarmService.java | 3 +- .../DefaultAlarmSubscriptionService.java | 2 +- .../controller/AlarmControllerTest.java | 30 +++- .../alarm/DefaultTbAlarmServiceTest.java | 163 ++++++++++++------ .../engine/api/RuleEngineAlarmService.java | 3 +- 8 files changed, 154 insertions(+), 86 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java index a83e9e446e..dd8d883145 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java @@ -157,7 +157,7 @@ public class AlarmController extends BaseController { @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/alarm/{alarmId}", method = RequestMethod.DELETE) @ResponseBody - public Boolean deleteAlarm(@Parameter(description = ALARM_ID_PARAM_DESCRIPTION) @PathVariable(ALARM_ID) String strAlarmId) throws ThingsboardException { + public boolean deleteAlarm(@Parameter(description = ALARM_ID_PARAM_DESCRIPTION) @PathVariable(ALARM_ID) String strAlarmId) throws ThingsboardException { checkParameter(ALARM_ID, strAlarmId); AlarmId alarmId = new AlarmId(toUUID(strAlarmId)); Alarm alarm = checkAlarmId(alarmId, Operation.DELETE); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index e9ee7202a7..476a4ef5ca 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -17,10 +17,8 @@ package org.thingsboard.server.service.entitiy; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.core.env.Environment; import org.thingsboard.server.cluster.TbClusterService; @@ -31,16 +29,10 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.dao.alarm.AlarmService; -import org.thingsboard.server.dao.asset.AssetProfileService; -import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; -import org.thingsboard.server.dao.device.DeviceProfileService; -import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; @@ -55,12 +47,6 @@ public abstract class AbstractTbEntityService { @Autowired private Environment env; - @Value("${server.log_controller_error_stack_trace}") - @Getter - private boolean logControllerErrorStackTrace; - - @Autowired - protected DbCallbackExecutorService dbExecutor; @Autowired(required = false) protected TbLogEntityActionService logEntityActionService; @Autowired(required = false) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java index 4766d8e039..5a1dee3bb5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java @@ -190,11 +190,24 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb } @Override - public Boolean delete(Alarm alarm, User user) { - TenantId tenantId = alarm.getTenantId(); - logEntityActionService.logEntityAction(tenantId, alarm.getOriginator(), alarm, alarm.getCustomerId(), - ActionType.ALARM_DELETE, user, alarm.getId()); - return alarmSubscriptionService.deleteAlarm(tenantId, alarm.getId()); + public boolean delete(Alarm alarm, User user) { + var tenantId = alarm.getTenantId(); + var alarmId = alarm.getId(); + var alarmOriginator = alarm.getOriginator(); + + boolean deleted; + try { + deleted = alarmSubscriptionService.deleteAlarm(tenantId, alarmId); + } catch (Exception e) { + logEntityActionService.logEntityAction(tenantId, emptyId(alarmOriginator.getEntityType()), ActionType.ALARM_DELETE, user, e, alarmId); + throw e; + } + + if (deleted) { + logEntityActionService.logEntityAction(tenantId, alarmOriginator, alarm, alarm.getCustomerId(), ActionType.ALARM_DELETE, user, alarmId); + } + + return deleted; } private static long getOrDefault(long ts) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java index 11b5c864ac..c034c39561 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java @@ -43,5 +43,6 @@ public interface TbAlarmService { void unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, List alarms, long unassignTs); - Boolean delete(Alarm alarm, User user); + boolean delete(Alarm alarm, User user); + } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java index 7a5a563e68..fd9d8b7141 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java @@ -115,7 +115,7 @@ public class DefaultAlarmSubscriptionService extends AbstractSubscriptionService } @Override - public Boolean deleteAlarm(TenantId tenantId, AlarmId alarmId) { + public boolean deleteAlarm(TenantId tenantId, AlarmId alarmId) { AlarmApiCallResult result = alarmService.delAlarm(tenantId, alarmId); onAlarmDeleted(result); return result.isSuccessful(); diff --git a/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java index d80c8a6ba5..8a49d34d20 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.controller; +import com.datastax.oss.driver.api.core.uuid.Uuids; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; @@ -57,6 +58,8 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.verifyNoInteractions; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @Slf4j @@ -308,6 +311,21 @@ public class AlarmControllerTest extends AbstractControllerTest { testNotifyEntityNever(alarm.getId(), alarm); } + @Test + public void testDeleteNonExistentAlarm() throws Exception { + loginTenantAdmin(); + + var nonExistentAlarmId = Uuids.timeBased(); + + Mockito.reset(tbClusterService, auditLogService); + + doDelete("/api/alarm/" + nonExistentAlarmId) + .andExpect(status().isNotFound()) + .andExpect(statusReason(is("Alarm with id [" + nonExistentAlarmId + "] is not found"))); + + verifyNoInteractions(tbClusterService, auditLogService); + } + @Test public void testClearAlarmViaCustomer() throws Exception { loginCustomerUser(); @@ -634,12 +652,12 @@ public class AlarmControllerTest extends AbstractControllerTest { doDelete("/api/user/" + savedUser.getId().getId()).andExpect(status().isOk()); - Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { - AlarmInfo alarmInfo = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class); - Assert.assertNotNull(alarmInfo); - Assert.assertNull(alarmInfo.getAssigneeId()); - Assert.assertTrue(alarmInfo.getAssignTs() >= afterAssignmentTs); - }); + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + AlarmInfo alarmInfo = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class); + Assert.assertNotNull(alarmInfo); + Assert.assertNull(alarmInfo.getAssigneeId()); + Assert.assertTrue(alarmInfo.getAssignTs() >= afterAssignmentTs); + }); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java index ac85b48bc1..670ab390ac 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java @@ -15,15 +15,12 @@ */ package org.thingsboard.server.service.entitiy.alarm; +import com.datastax.oss.driver.api.core.uuid.Uuids; import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.extern.slf4j.Slf4j; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.boot.test.mock.mockito.SpyBean; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.User; @@ -35,6 +32,9 @@ import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.alarm.AlarmService; @@ -47,7 +47,6 @@ import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.entitiy.TbLogEntityActionService; -import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; @@ -55,58 +54,57 @@ import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import java.util.List; import java.util.UUID; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; -@Slf4j -@RunWith(SpringRunner.class) -@ContextConfiguration(classes = DefaultTbAlarmService.class) -@TestPropertySource(properties = { - "server.log_controller_error_stack_trace=false" -}) -public class DefaultTbAlarmServiceTest { +@SpringJUnitConfig(DefaultTbAlarmService.class) +class DefaultTbAlarmServiceTest { @MockBean - protected DbCallbackExecutorService dbExecutor; + TbLogEntityActionService logEntityActionService; @MockBean - protected TbLogEntityActionService logEntityActionService; + EdgeService edgeService; @MockBean - protected EdgeService edgeService; + AlarmService alarmService; @MockBean - protected AlarmService alarmService; + TbAlarmCommentService alarmCommentService; @MockBean - protected TbAlarmCommentService alarmCommentService; + AlarmSubscriptionService alarmSubscriptionService; @MockBean - protected AlarmSubscriptionService alarmSubscriptionService; + CustomerService customerService; @MockBean - protected CustomerService customerService; + TbClusterService tbClusterService; @MockBean - protected TbClusterService tbClusterService; + EntitiesVersionControlService vcService; @MockBean - private EntitiesVersionControlService vcService; + AccessControlService accessControlService; @MockBean - private AccessControlService accessControlService; + TenantService tenantService; @MockBean - private TenantService tenantService; + AssetService assetService; @MockBean - private AssetService assetService; + DeviceService deviceService; @MockBean - private DeviceService deviceService; + AssetProfileService assetProfileService; @MockBean - private AssetProfileService assetProfileService; + DeviceProfileService deviceProfileService; @MockBean - private DeviceProfileService deviceProfileService; - @MockBean - private EntityService entityService; - @SpyBean + EntityService entityService; + + @Autowired DefaultTbAlarmService service; + TenantId tenantId = TenantId.fromUUID(Uuids.timeBased()); + CustomerId customerId = new CustomerId(Uuids.timeBased()); + @Test - public void testSave() throws ThingsboardException { + void testSave() throws ThingsboardException { var alarm = new AlarmInfo(); when(alarmSubscriptionService.createAlarm(any())).thenReturn(AlarmApiCallResult.builder() .successful(true) @@ -115,45 +113,99 @@ public class DefaultTbAlarmServiceTest { .build()); service.save(alarm, new User()); - verify(logEntityActionService, times(1)).logEntityAction(any(), any(), any(), any(), eq(ActionType.ADDED), any()); - verify(alarmSubscriptionService, times(1)).createAlarm(any()); + verify(logEntityActionService).logEntityAction(any(), any(), any(), any(), eq(ActionType.ADDED), any()); + verify(alarmSubscriptionService).createAlarm(any()); } @Test - public void testAck() throws ThingsboardException { + void testAck() throws ThingsboardException { var alarm = new Alarm(); when(alarmSubscriptionService.acknowledgeAlarm(any(), any(), anyLong())) .thenReturn(AlarmApiCallResult.builder().successful(true).modified(true).alarm(new AlarmInfo()).build()); service.ack(alarm, new User(new UserId(UUID.randomUUID()))); - verify(alarmCommentService, times(1)).saveAlarmComment(any(), any(), any()); - verify(logEntityActionService, times(1)).logEntityAction(any(), any(), any(), any(), eq(ActionType.ALARM_ACK), any()); - verify(alarmSubscriptionService, times(1)).acknowledgeAlarm(any(), any(), anyLong()); + verify(alarmCommentService).saveAlarmComment(any(), any(), any()); + verify(logEntityActionService).logEntityAction(any(), any(), any(), any(), eq(ActionType.ALARM_ACK), any()); + verify(alarmSubscriptionService).acknowledgeAlarm(any(), any(), anyLong()); } @Test - public void testClear() throws ThingsboardException { + void testClear() throws ThingsboardException { var alarm = new Alarm(); alarm.setAcknowledged(true); when(alarmSubscriptionService.clearAlarm(any(), any(), anyLong(), any())) .thenReturn(AlarmApiCallResult.builder().successful(true).cleared(true).alarm(new AlarmInfo()).build()); service.clear(alarm, new User(new UserId(UUID.randomUUID()))); - verify(alarmCommentService, times(1)).saveAlarmComment(any(), any(), any()); - verify(logEntityActionService, times(1)).logEntityAction(any(), any(), any(), any(), eq(ActionType.ALARM_CLEAR), any()); - verify(alarmSubscriptionService, times(1)).clearAlarm(any(), any(), anyLong(), any()); + verify(alarmCommentService).saveAlarmComment(any(), any(), any()); + verify(logEntityActionService).logEntityAction(any(), any(), any(), any(), eq(ActionType.ALARM_CLEAR), any()); + verify(alarmSubscriptionService).clearAlarm(any(), any(), anyLong(), any()); } @Test - public void testDelete() { - service.delete(new Alarm(), new User()); + void testDelete_deleteApiReturnsTrue_shouldLogActionAndReturnTrue() { + // GIVEN + var alarmOriginator = new DeviceId(Uuids.timeBased()); + + var alarm = new Alarm(new AlarmId(Uuids.timeBased())); + alarm.setTenantId(tenantId); + alarm.setCustomerId(customerId); + alarm.setOriginator(alarmOriginator); + + var user = new User(); + + when(alarmSubscriptionService.deleteAlarm(tenantId, alarm.getId())).thenReturn(true); - verify(logEntityActionService, times(1)).logEntityAction(any(), any(), any(), any(), eq(ActionType.ALARM_DELETE), any(), any()); - verify(alarmSubscriptionService, times(1)).deleteAlarm(any(), any()); + // WHEN + boolean actual = service.delete(alarm, user); + + assertThat(actual).isTrue(); + verify(logEntityActionService).logEntityAction(tenantId, alarmOriginator, alarm, alarm.getCustomerId(), ActionType.ALARM_DELETE, user, alarm.getId()); + verify(alarmSubscriptionService).deleteAlarm(tenantId, alarm.getId()); } @Test - public void testUnassignAlarm() throws ThingsboardException { + void testDelete_deleteApiReturnsFalse_shouldNotLogActionAndReturnFalse() { + // GIVEN + var alarm = new Alarm(new AlarmId(Uuids.timeBased())); + alarm.setTenantId(tenantId); + + var user = new User(); + + // WHEN + boolean actual = service.delete(alarm, user); + + assertThat(actual).isFalse(); + verifyNoInteractions(logEntityActionService); + verify(alarmSubscriptionService).deleteAlarm(tenantId, alarm.getId()); + } + + @Test + void testDelete_deleteApiThrowsException_shouldLogFailedActionAndRethrow() { + // GIVEN + var alarmOriginator = new DeviceId(Uuids.timeBased()); + + var alarm = new Alarm(new AlarmId(Uuids.timeBased())); + alarm.setTenantId(tenantId); + alarm.setOriginator(alarmOriginator); + + var user = new User(); + + var exception = new RuntimeException("failed to delete alarm"); + + when(alarmSubscriptionService.deleteAlarm(tenantId, alarm.getId())).thenThrow(exception); + + // WHEN-THEN + assertThatThrownBy(() -> service.delete(alarm, user)) + .isInstanceOf(RuntimeException.class) + .hasMessage("failed to delete alarm"); + + verify(logEntityActionService).logEntityAction(tenantId, new DeviceId(EntityId.NULL_UUID), ActionType.ALARM_DELETE, user, exception, alarm.getId()); + verify(alarmSubscriptionService).deleteAlarm(tenantId, alarm.getId()); + } + + @Test + void testUnassignAlarm() throws ThingsboardException { AlarmInfo alarm = new AlarmInfo(); alarm.setId(new AlarmId(UUID.randomUUID())); when(alarmSubscriptionService.unassignAlarm(any(), any(), anyLong())) @@ -174,12 +226,11 @@ public class DefaultTbAlarmServiceTest { .comment(commentNode) .build(); - verify(alarmCommentService, times(1)) - .saveAlarmComment(eq(alarm), eq(expectedAlarmComment), eq(user)); + verify(alarmCommentService).saveAlarmComment(eq(alarm), eq(expectedAlarmComment), eq(user)); } @Test - public void testUnassignDeletedUserAlarms() throws ThingsboardException { + void testUnassignDeletedUserAlarms() throws ThingsboardException { AlarmInfo alarm = new AlarmInfo(); alarm.setId(new AlarmId(UUID.randomUUID())); @@ -189,7 +240,7 @@ public class DefaultTbAlarmServiceTest { User user = new User(); user.setEmail("testEmail@gmail.com"); user.setId(new UserId(UUID.randomUUID())); - service.unassignDeletedUserAlarms(new TenantId(UUID.randomUUID()), user.getId(), user.getTitle(), List.of(alarm.getUuidId()), System.currentTimeMillis()); + service.unassignDeletedUserAlarms(tenantId, user.getId(), user.getTitle(), List.of(alarm.getUuidId()), System.currentTimeMillis()); ObjectNode commentNode = JacksonUtil.newObjectNode(); commentNode.put("subtype", "ASSIGN"); @@ -200,9 +251,7 @@ public class DefaultTbAlarmServiceTest { .comment(commentNode) .build(); - verify(alarmCommentService, times(1)) - .saveAlarmComment(eq(alarm), eq(expectedAlarmComment), eq(null)); + verify(alarmCommentService).saveAlarmComment(eq(alarm), eq(expectedAlarmComment), eq(null)); } - } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineAlarmService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineAlarmService.java index 20f2342ebe..48fda3b781 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineAlarmService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineAlarmService.java @@ -70,7 +70,7 @@ public interface RuleEngineAlarmService { AlarmApiCallResult unassignAlarm(TenantId tenantId, AlarmId alarmId, long assignTs); // Other API - Boolean deleteAlarm(TenantId tenantId, AlarmId alarmId); + boolean deleteAlarm(TenantId tenantId, AlarmId alarmId); ListenableFuture findAlarmByIdAsync(TenantId tenantId, AlarmId alarmId); @@ -99,4 +99,5 @@ public interface RuleEngineAlarmService { PageData findAlarmDataByQueryForEntities(TenantId tenantId, AlarmDataQuery query, Collection orderedEntityIds); PageData findAlarmTypesByTenantId(TenantId tenantId, PageLink pageLink); + } From 65f9adcc650cd58766f6d905d889e1b150fd6077 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 28 Apr 2025 11:03:33 +0300 Subject: [PATCH 23/40] added trendz settings --- .../controller/SystemInfoController.java | 5 ++ .../server/controller/TrendzController.java | 80 +++++++++++++++++++ .../permission/CustomerUserPermissions.java | 1 + .../service/security/permission/Resource.java | 3 +- .../permission/SysAdminPermissions.java | 3 +- .../permission/TenantAdminPermissions.java | 1 + .../src/main/resources/thingsboard.yml | 3 + .../controller/TrendzControllerTest.java | 63 +++++++++++++++ .../dao/trendz/TrendzSettingsService.java | 27 +++++++ .../server/common/data/CacheConstants.java | 1 + .../server/common/data/SystemParams.java | 2 + .../common/data/trendz/TrendzSettings.java | 26 ++++++ .../trendz/DefaultTrendzSettingsService.java | 63 +++++++++++++++ 13 files changed, 276 insertions(+), 2 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/controller/TrendzController.java create mode 100644 application/src/test/java/org/thingsboard/server/controller/TrendzControllerTest.java create mode 100644 common/dao-api/src/main/java/org/thingsboard/server/dao/trendz/TrendzSettingsService.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/trendz/TrendzSettings.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/trendz/DefaultTrendzSettingsService.java diff --git a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java index 4ee86871d6..9ecb9dfcc9 100644 --- a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java +++ b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java @@ -42,6 +42,7 @@ import org.thingsboard.server.common.data.settings.UserSettings; import org.thingsboard.server.common.data.settings.UserSettingsType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.mobile.QrCodeSettingService; +import org.thingsboard.server.dao.trendz.TrendzSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; @@ -87,6 +88,9 @@ public class SystemInfoController extends BaseController { @Autowired private DebugModeRateLimitsConfig debugModeRateLimitsConfig; + @Autowired + private TrendzSettingsService trendzSettingsService; + @PostConstruct public void init() { JsonNode info = buildInfoObject(); @@ -162,6 +166,7 @@ public class SystemInfoController extends BaseController { systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID)) .map(QrCodeSettings::getQrCodeConfig).map(QRCodeConfig::isShowOnHomePage) .orElse(false)); + systemParams.setTrendzSettings(trendzSettingsService.findTrendzSettings(currentUser.getTenantId())); return systemParams; } diff --git a/application/src/main/java/org/thingsboard/server/controller/TrendzController.java b/application/src/main/java/org/thingsboard/server/controller/TrendzController.java new file mode 100644 index 0000000000..4430f740c7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/TrendzController.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import lombok.RequiredArgsConstructor; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.trendz.TrendzSettings; +import org.thingsboard.server.config.annotations.ApiOperation; +import org.thingsboard.server.dao.trendz.TrendzSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; + +import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END; +import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START; +import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; +import static org.thingsboard.server.controller.ControllerConstants.SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH; + +@RestController +@TbCoreComponent +@RequiredArgsConstructor +@RequestMapping("/api") +public class TrendzController extends BaseController { + + private final TrendzSettingsService trendzSettingsService; + + @ApiOperation(value = "Save Trendz settings (saveTrendzSettings)", + notes = "Saves Trendz settings for this tenant or sysadmin.\n" + NEW_LINE + + "Here is an example of the Trendz settings:\n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + + " \"enabled\": true,\n" + + " \"trendzUrl\": \"https://some.domain.com:18888/also_necessary_prefix\"\n" + + "}" + + MARKDOWN_CODE_BLOCK_END + + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH) + @PostMapping("/trendz/settings") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") + public TrendzSettings saveTrendzSettings(@RequestBody TrendzSettings trendzSettings, + @AuthenticationPrincipal SecurityUser user) throws ThingsboardException { + accessControlService.checkPermission(user, Resource.TRENDZ_SETTINGS, Operation.WRITE); + TenantId tenantId = user.isSystemAdmin() ? TenantId.SYS_TENANT_ID : user.getTenantId(); + trendzSettingsService.saveTrendzSettings(tenantId, trendzSettings); + return trendzSettings; + } + + @ApiOperation(value = "Get Trendz Settings (getTrendzSettings)", + notes = "Retrieves trendz settings for this tenant or sysadmin." + + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH) + @GetMapping("/trendz/settings") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") + public TrendzSettings getTrendzSettings(@AuthenticationPrincipal SecurityUser user) throws ThingsboardException { + accessControlService.checkPermission(user, Resource.TRENDZ_SETTINGS, Operation.READ); + TenantId tenantId = user.isSystemAdmin() ? TenantId.SYS_TENANT_ID : user.getTenantId(); + return trendzSettingsService.findTrendzSettings(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java index 8124671cd7..ea7457ec47 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java @@ -48,6 +48,7 @@ public class CustomerUserPermissions extends AbstractPermissions { put(Resource.ASSET_PROFILE, profilePermissionChecker); put(Resource.TB_RESOURCE, customerResourcePermissionChecker); put(Resource.MOBILE_APP_SETTINGS, new PermissionChecker.GenericPermissionChecker(Operation.READ)); + put(Resource.TRENDZ_SETTINGS, new PermissionChecker.GenericPermissionChecker(Operation.READ)); } private static final PermissionChecker customerAlarmPermissionChecker = new PermissionChecker() { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java index 9d7590f786..1added11b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java @@ -51,7 +51,8 @@ public enum Resource { NOTIFICATION(EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE), MOBILE_APP_SETTINGS, - CALCULATED_FIELD(EntityType.CALCULATED_FIELD); + CALCULATED_FIELD(EntityType.CALCULATED_FIELD), + TRENDZ_SETTINGS; private final Set entityTypes; diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java index e64f5b49dd..acfec9da25 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.service.security.model.SecurityUser; -@Component(value="sysAdminPermissions") +@Component(value = "sysAdminPermissions") public class SysAdminPermissions extends AbstractPermissions { public SysAdminPermissions() { @@ -45,6 +45,7 @@ public class SysAdminPermissions extends AbstractPermissions { put(Resource.QUEUE, systemEntityPermissionChecker); put(Resource.NOTIFICATION, systemEntityPermissionChecker); put(Resource.MOBILE_APP_SETTINGS, PermissionChecker.allowAllPermissionChecker); + put(Resource.TRENDZ_SETTINGS, PermissionChecker.allowAllPermissionChecker); } private static final PermissionChecker systemEntityPermissionChecker = new PermissionChecker() { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index a072cf2738..35824d7858 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -56,6 +56,7 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.MOBILE_APP, tenantEntityPermissionChecker); put(Resource.MOBILE_APP_BUNDLE, tenantEntityPermissionChecker); put(Resource.CALCULATED_FIELD, tenantEntityPermissionChecker); + put(Resource.TRENDZ_SETTINGS, PermissionChecker.allowAllPermissionChecker); } public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 418263076b..a8050b00ed 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -644,6 +644,9 @@ cache: mobileSecretKey: timeToLiveInMinutes: "${CACHE_MOBILE_SECRET_KEY_TTL:2}" # QR secret key cache TTL maxSize: "${CACHE_MOBILE_SECRET_KEY_MAX_SIZE:10000}" # 0 means the cache is disabled + trendzSettings: + timeToLiveInMinutes: "${CACHE_SPECS_TRENDZ_SETTINGS_TTL:1440}" # Trendz settings cache TTL + maxSize: "${CACHE_SPECS_TRENDZ_SETTINGS_MAX_SIZE:10000}" # 0 means the cache is disabled # Deliberately placed outside the 'specs' group above notificationRules: diff --git a/application/src/test/java/org/thingsboard/server/controller/TrendzControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TrendzControllerTest.java new file mode 100644 index 0000000000..8cf8d0f90c --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/TrendzControllerTest.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import org.junit.Test; +import org.thingsboard.server.common.data.trendz.TrendzSettings; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@DaoSqlTest +public class TrendzControllerTest extends AbstractControllerTest { + + private final String trendzUrl = "https://some.domain.com:18888/also_necessary_prefix"; + + @Test + public void testTrendzSettingsWhenTenant() throws Exception { + loginTenantAdmin(); + + TrendzSettings trendzSettings = doGet("/api/trendz/settings", TrendzSettings.class); + + assertThat(trendzSettings).isNotNull(); + assertThat(trendzSettings.isEnabled()).isFalse(); + assertThat(trendzSettings.getTrendzUrl()).isNull(); + + trendzSettings.setEnabled(true); + trendzSettings.setTrendzUrl(trendzUrl); + + doPost("/api/trendz/settings", trendzSettings).andExpect(status().isOk()); + + TrendzSettings updatedTrendzSettings = doGet("/api/trendz/settings", TrendzSettings.class); + assertThat(updatedTrendzSettings).isEqualTo(trendzSettings); + } + + @Test + public void testTrendzSettingsWhenCustomer() throws Exception { + loginCustomerUser(); + + TrendzSettings trendzSettings = new TrendzSettings(); + trendzSettings.setEnabled(true); + trendzSettings.setTrendzUrl("https://some.domain.com:18888/customer_trendz"); + + doPost("/api/trendz/settings", trendzSettings).andExpect(status().isForbidden()); + + TrendzSettings fetchedTrendzSettings = doGet("/api/trendz/settings", TrendzSettings.class); + assertThat(fetchedTrendzSettings).isNotNull(); + } + +} \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/trendz/TrendzSettingsService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/trendz/TrendzSettingsService.java new file mode 100644 index 0000000000..6f66054518 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/trendz/TrendzSettingsService.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.trendz; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.trendz.TrendzSettings; + +public interface TrendzSettingsService { + + void saveTrendzSettings(TenantId tenantId, TrendzSettings settings); + + TrendzSettings findTrendzSettings(TenantId tenantId); + +} \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java index 35f69e1544..5b167c88a2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java @@ -35,6 +35,7 @@ public class CacheConstants { public static final String DEVICE_PROFILE_CACHE = "deviceProfiles"; public static final String NOTIFICATION_SETTINGS_CACHE = "notificationSettings"; public static final String SENT_NOTIFICATIONS_CACHE = "sentNotifications"; + public static final String TRENDZ_SETTINGS_CACHE = "trendzSettings"; public static final String ASSET_PROFILE_CACHE = "assetProfiles"; public static final String ATTRIBUTES_CACHE = "attributes"; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java index b1ef4d7f22..fe3eb4e4d8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java @@ -17,6 +17,7 @@ package org.thingsboard.server.common.data; import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; +import org.thingsboard.server.common.data.trendz.TrendzSettings; import java.util.List; @@ -37,4 +38,5 @@ public class SystemParams { String calculatedFieldDebugPerTenantLimitsConfiguration; long maxArgumentsPerCF; long maxDataPointsPerRollingArg; + TrendzSettings trendzSettings; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/trendz/TrendzSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/trendz/TrendzSettings.java new file mode 100644 index 0000000000..dfd1967740 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/trendz/TrendzSettings.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.trendz; + +import lombok.Data; + +@Data +public class TrendzSettings { + + private boolean enabled; + private String trendzUrl; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/trendz/DefaultTrendzSettingsService.java b/dao/src/main/java/org/thingsboard/server/dao/trendz/DefaultTrendzSettingsService.java new file mode 100644 index 0000000000..98df523245 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/trendz/DefaultTrendzSettingsService.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.trendz; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.trendz.TrendzSettings; +import org.thingsboard.server.dao.settings.AdminSettingsService; + +import java.util.Optional; + +@Service +@RequiredArgsConstructor +@Slf4j +public class DefaultTrendzSettingsService implements TrendzSettingsService { + + private final AdminSettingsService adminSettingsService; + + private static final String SETTINGS_KEY = "trendz"; + + @CacheEvict(cacheNames = CacheConstants.TRENDZ_SETTINGS_CACHE, key = "#tenantId") + @Override + public void saveTrendzSettings(TenantId tenantId, TrendzSettings settings) { + AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByTenantIdAndKey(tenantId, SETTINGS_KEY)) + .orElseGet(() -> { + AdminSettings newAdminSettings = new AdminSettings(); + newAdminSettings.setTenantId(tenantId); + newAdminSettings.setKey(SETTINGS_KEY); + return newAdminSettings; + }); + adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); + adminSettingsService.saveAdminSettings(tenantId, adminSettings); + } + + @Cacheable(cacheNames = CacheConstants.TRENDZ_SETTINGS_CACHE, key = "#tenantId") + @Override + public TrendzSettings findTrendzSettings(TenantId tenantId) { + return Optional.ofNullable(adminSettingsService.findAdminSettingsByTenantIdAndKey(tenantId, SETTINGS_KEY)) + .map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), TrendzSettings.class)) + .orElseGet(TrendzSettings::new); + } + +} From 85dfbe8792236288b9f61c25004fbeff7c2e7c3e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 28 Apr 2025 11:08:26 +0300 Subject: [PATCH 24/40] added new lines to the end of the files --- .../org/thingsboard/server/controller/TrendzControllerTest.java | 2 +- .../thingsboard/server/dao/trendz/TrendzSettingsService.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/TrendzControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TrendzControllerTest.java index 8cf8d0f90c..e36861924c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TrendzControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TrendzControllerTest.java @@ -60,4 +60,4 @@ public class TrendzControllerTest extends AbstractControllerTest { assertThat(fetchedTrendzSettings).isNotNull(); } -} \ No newline at end of file +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/trendz/TrendzSettingsService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/trendz/TrendzSettingsService.java index 6f66054518..b79c4d144d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/trendz/TrendzSettingsService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/trendz/TrendzSettingsService.java @@ -24,4 +24,4 @@ public interface TrendzSettingsService { TrendzSettings findTrendzSettings(TenantId tenantId); -} \ No newline at end of file +} From b09967074e739b57e2802828e8909c11809a7b0f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 28 Apr 2025 11:20:05 +0300 Subject: [PATCH 25/40] added delete method --- .../server/dao/trendz/TrendzSettingsService.java | 2 ++ .../thingsboard/server/dao/tenant/TenantServiceImpl.java | 4 ++++ .../server/dao/trendz/DefaultTrendzSettingsService.java | 6 ++++++ 3 files changed, 12 insertions(+) diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/trendz/TrendzSettingsService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/trendz/TrendzSettingsService.java index b79c4d144d..31f9495345 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/trendz/TrendzSettingsService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/trendz/TrendzSettingsService.java @@ -24,4 +24,6 @@ public interface TrendzSettingsService { TrendzSettings findTrendzSettings(TenantId tenantId); + void deleteTrendzSettings(TenantId tenantId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 1dbca5af12..e35c9269b3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -44,6 +44,7 @@ import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; import org.thingsboard.server.dao.service.validator.TenantDataValidator; import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.dao.trendz.TrendzSettingsService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.dao.user.UserService; @@ -81,6 +82,8 @@ public class TenantServiceImpl extends AbstractCachedEntityService existsTenantCache; @@ -166,6 +169,7 @@ public class TenantServiceImpl extends AbstractCachedEntityService Date: Mon, 28 Apr 2025 12:10:11 +0300 Subject: [PATCH 26/40] fixes --- .../controller/SystemInfoController.java | 2 +- .../server/controller/TrendzController.java | 26 ++++++++-------- .../permission/CustomerUserPermissions.java | 1 - .../service/security/permission/Resource.java | 3 +- .../permission/SysAdminPermissions.java | 1 - .../permission/TenantAdminPermissions.java | 1 - .../controller/TrendzControllerTest.java | 30 ++++++++++++++----- .../common/data/trendz/TrendzSettings.java | 2 +- .../server/dao/tenant/TenantServiceImpl.java | 4 +-- 9 files changed, 40 insertions(+), 30 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java index 9ecb9dfcc9..29f4daa783 100644 --- a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java +++ b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java @@ -162,11 +162,11 @@ public class SystemInfoController extends BaseController { } systemParams.setMaxArgumentsPerCF(tenantProfileConfiguration.getMaxArgumentsPerCF()); systemParams.setMaxDataPointsPerRollingArg(tenantProfileConfiguration.getMaxDataPointsPerRollingArg()); + systemParams.setTrendzSettings(trendzSettingsService.findTrendzSettings(currentUser.getTenantId())); } systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID)) .map(QrCodeSettings::getQrCodeConfig).map(QRCodeConfig::isShowOnHomePage) .orElse(false)); - systemParams.setTrendzSettings(trendzSettingsService.findTrendzSettings(currentUser.getTenantId())); return systemParams; } diff --git a/application/src/main/java/org/thingsboard/server/controller/TrendzController.java b/application/src/main/java/org/thingsboard/server/controller/TrendzController.java index 4430f740c7..8261071670 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TrendzController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TrendzController.java @@ -36,7 +36,8 @@ import org.thingsboard.server.service.security.permission.Resource; import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END; import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START; import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; -import static org.thingsboard.server.controller.ControllerConstants.SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; @RestController @TbCoreComponent @@ -47,33 +48,32 @@ public class TrendzController extends BaseController { private final TrendzSettingsService trendzSettingsService; @ApiOperation(value = "Save Trendz settings (saveTrendzSettings)", - notes = "Saves Trendz settings for this tenant or sysadmin.\n" + NEW_LINE + + notes = "Saves Trendz settings for this tenant.\n" + NEW_LINE + "Here is an example of the Trendz settings:\n" + MARKDOWN_CODE_BLOCK_START + "{\n" + " \"enabled\": true,\n" + - " \"trendzUrl\": \"https://some.domain.com:18888/also_necessary_prefix\"\n" + + " \"baseUrl\": \"https://some.domain.com:18888/also_necessary_prefix\"\n" + "}" + MARKDOWN_CODE_BLOCK_END + - SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH) + TENANT_AUTHORITY_PARAGRAPH) @PostMapping("/trendz/settings") - @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") public TrendzSettings saveTrendzSettings(@RequestBody TrendzSettings trendzSettings, @AuthenticationPrincipal SecurityUser user) throws ThingsboardException { - accessControlService.checkPermission(user, Resource.TRENDZ_SETTINGS, Operation.WRITE); - TenantId tenantId = user.isSystemAdmin() ? TenantId.SYS_TENANT_ID : user.getTenantId(); + accessControlService.checkPermission(user, Resource.ADMIN_SETTINGS, Operation.WRITE); + TenantId tenantId = user.getTenantId(); trendzSettingsService.saveTrendzSettings(tenantId, trendzSettings); return trendzSettings; } @ApiOperation(value = "Get Trendz Settings (getTrendzSettings)", - notes = "Retrieves trendz settings for this tenant or sysadmin." + - SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH) + notes = "Retrieves Trendz settings for this tenant." + + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) @GetMapping("/trendz/settings") - @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") - public TrendzSettings getTrendzSettings(@AuthenticationPrincipal SecurityUser user) throws ThingsboardException { - accessControlService.checkPermission(user, Resource.TRENDZ_SETTINGS, Operation.READ); - TenantId tenantId = user.isSystemAdmin() ? TenantId.SYS_TENANT_ID : user.getTenantId(); + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") + public TrendzSettings getTrendzSettings(@AuthenticationPrincipal SecurityUser user) { + TenantId tenantId = user.getTenantId(); return trendzSettingsService.findTrendzSettings(tenantId); } diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java index ea7457ec47..8124671cd7 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java @@ -48,7 +48,6 @@ public class CustomerUserPermissions extends AbstractPermissions { put(Resource.ASSET_PROFILE, profilePermissionChecker); put(Resource.TB_RESOURCE, customerResourcePermissionChecker); put(Resource.MOBILE_APP_SETTINGS, new PermissionChecker.GenericPermissionChecker(Operation.READ)); - put(Resource.TRENDZ_SETTINGS, new PermissionChecker.GenericPermissionChecker(Operation.READ)); } private static final PermissionChecker customerAlarmPermissionChecker = new PermissionChecker() { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java index 1added11b0..9d7590f786 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java @@ -51,8 +51,7 @@ public enum Resource { NOTIFICATION(EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE), MOBILE_APP_SETTINGS, - CALCULATED_FIELD(EntityType.CALCULATED_FIELD), - TRENDZ_SETTINGS; + CALCULATED_FIELD(EntityType.CALCULATED_FIELD); private final Set entityTypes; diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java index acfec9da25..6bd7aacf54 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java @@ -45,7 +45,6 @@ public class SysAdminPermissions extends AbstractPermissions { put(Resource.QUEUE, systemEntityPermissionChecker); put(Resource.NOTIFICATION, systemEntityPermissionChecker); put(Resource.MOBILE_APP_SETTINGS, PermissionChecker.allowAllPermissionChecker); - put(Resource.TRENDZ_SETTINGS, PermissionChecker.allowAllPermissionChecker); } private static final PermissionChecker systemEntityPermissionChecker = new PermissionChecker() { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 35824d7858..a072cf2738 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -56,7 +56,6 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.MOBILE_APP, tenantEntityPermissionChecker); put(Resource.MOBILE_APP_BUNDLE, tenantEntityPermissionChecker); put(Resource.CALCULATED_FIELD, tenantEntityPermissionChecker); - put(Resource.TRENDZ_SETTINGS, PermissionChecker.allowAllPermissionChecker); } public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { diff --git a/application/src/test/java/org/thingsboard/server/controller/TrendzControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TrendzControllerTest.java index e36861924c..115b895521 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TrendzControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TrendzControllerTest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.controller; +import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.trendz.TrendzSettings; import org.thingsboard.server.dao.service.DaoSqlTest; @@ -27,6 +28,17 @@ public class TrendzControllerTest extends AbstractControllerTest { private final String trendzUrl = "https://some.domain.com:18888/also_necessary_prefix"; + @Before + public void setUp() throws Exception { + loginTenantAdmin(); + + TrendzSettings trendzSettings = new TrendzSettings(); + trendzSettings.setEnabled(true); + trendzSettings.setBaseUrl(trendzUrl); + + doPost("/api/trendz/settings", trendzSettings).andExpect(status().isOk()); + } + @Test public void testTrendzSettingsWhenTenant() throws Exception { loginTenantAdmin(); @@ -34,11 +46,11 @@ public class TrendzControllerTest extends AbstractControllerTest { TrendzSettings trendzSettings = doGet("/api/trendz/settings", TrendzSettings.class); assertThat(trendzSettings).isNotNull(); - assertThat(trendzSettings.isEnabled()).isFalse(); - assertThat(trendzSettings.getTrendzUrl()).isNull(); + assertThat(trendzSettings.isEnabled()).isTrue(); + assertThat(trendzSettings.getBaseUrl()).isEqualTo(trendzUrl); - trendzSettings.setEnabled(true); - trendzSettings.setTrendzUrl(trendzUrl); + String updatedUrl = "https://some.domain.com:18888/tenant_trendz"; + trendzSettings.setBaseUrl(updatedUrl); doPost("/api/trendz/settings", trendzSettings).andExpect(status().isOk()); @@ -50,14 +62,16 @@ public class TrendzControllerTest extends AbstractControllerTest { public void testTrendzSettingsWhenCustomer() throws Exception { loginCustomerUser(); - TrendzSettings trendzSettings = new TrendzSettings(); - trendzSettings.setEnabled(true); - trendzSettings.setTrendzUrl("https://some.domain.com:18888/customer_trendz"); + TrendzSettings newTrendzSettings = new TrendzSettings(); + newTrendzSettings.setEnabled(true); + newTrendzSettings.setBaseUrl("https://some.domain.com:18888/customer_trendz"); - doPost("/api/trendz/settings", trendzSettings).andExpect(status().isForbidden()); + doPost("/api/trendz/settings", newTrendzSettings).andExpect(status().isForbidden()); TrendzSettings fetchedTrendzSettings = doGet("/api/trendz/settings", TrendzSettings.class); assertThat(fetchedTrendzSettings).isNotNull(); + assertThat(fetchedTrendzSettings.isEnabled()).isTrue(); + assertThat(fetchedTrendzSettings.getBaseUrl()).isEqualTo(trendzUrl); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/trendz/TrendzSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/trendz/TrendzSettings.java index dfd1967740..3c3b49399c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/trendz/TrendzSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/trendz/TrendzSettings.java @@ -21,6 +21,6 @@ import lombok.Data; public class TrendzSettings { private boolean enabled; - private String trendzUrl; + private String baseUrl; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index e35c9269b3..8c40ca3e14 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -166,10 +166,10 @@ public class TenantServiceImpl extends AbstractCachedEntityService INCORRECT_TENANT_ID + id); userService.deleteAllByTenantId(tenantId); - adminSettingsService.deleteAdminSettingsByTenantId(tenantId); - qrCodeSettingService.deleteByTenantId(tenantId); notificationSettingsService.deleteNotificationSettings(tenantId); trendzSettingsService.deleteTrendzSettings(tenantId); + adminSettingsService.deleteAdminSettingsByTenantId(tenantId); + qrCodeSettingService.deleteByTenantId(tenantId); tenantDao.removeById(tenantId, tenantId.getId()); publishEvictEvent(new TenantEvictEvent(tenantId, true)); From 8e9cd196c140e3acdcfdab06ebaf1d3af671af21 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 28 Apr 2025 14:23:05 +0300 Subject: [PATCH 27/40] fixed tests --- dao/src/test/resources/application-test.properties | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dao/src/test/resources/application-test.properties b/dao/src/test/resources/application-test.properties index a1bb335ad0..a44303107c 100644 --- a/dao/src/test/resources/application-test.properties +++ b/dao/src/test/resources/application-test.properties @@ -108,6 +108,9 @@ cache.specs.qrCodeSettings.maxSize=10000 cache.specs.mobileSecretKey.timeToLiveInMinutes=1440 cache.specs.mobileSecretKey.maxSize=10000 +cache.specs.trendzSettings.timeToLiveInMinutes=1440 +cache.specs.trendzSettings.maxSize=10000 + redis.connection.host=localhost redis.connection.port=6379 redis.connection.db=0 From 8c178a26477bd3c6960143586441804da617f42a Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 30 Apr 2025 16:46:12 +0300 Subject: [PATCH 28/40] UI: Fixed incorrect help links for calculated fields --- ui-ngx/src/app/shared/models/constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index e1306e577c..af576882b8 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -198,7 +198,7 @@ export const HelpLinks = { mobileApplication: `${helpBaseUrl}/docs${docPlatformPrefix}/mobile-center/applications/`, mobileBundle: `${helpBaseUrl}/docs${docPlatformPrefix}/mobile-center/mobile-center/`, mobileQrCode: `${helpBaseUrl}/docs${docPlatformPrefix}/user-guide/ui/mobile-qr-code/`, - calculatedField: `${helpBaseUrl}/docs${docPlatformPrefix}/`, + calculatedField: `${helpBaseUrl}/docs${docPlatformPrefix}/user-guide/calculated-fields/`, timewindowSettings: `${helpBaseUrl}/docs${docPlatformPrefix}/user-guide/dashboards/#time-window`, } }; From ff3c1e27ed83336a9d3d46b9c748d4ae562993a7 Mon Sep 17 00:00:00 2001 From: Vladyslav Prykhodko Date: Wed, 30 Apr 2025 23:47:19 +0300 Subject: [PATCH 29/40] =?UTF-8?q?UI:=20Improved=20Nederlands=20(Belgi?= =?UTF-8?q?=C3=AB)=20translation=20remove=20duplicate=20tranlate=20and=20i?= =?UTF-8?q?nvalid=20text?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../assets/locale/locale.constant-nl_BE.json | 34 +++---------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/ui-ngx/src/assets/locale/locale.constant-nl_BE.json b/ui-ngx/src/assets/locale/locale.constant-nl_BE.json index 348eea95b0..e64970bca3 100644 --- a/ui-ngx/src/assets/locale/locale.constant-nl_BE.json +++ b/ui-ngx/src/assets/locale/locale.constant-nl_BE.json @@ -59,8 +59,6 @@ "import": "Importeren", "export": "Exporteren", "share-via": "Delen via {{provider}}", - "move": "Verplaatsen", - "select": "Selecteren", "continue": "Voortzetten", "discard-changes": "Wijzigingen negeren", "download": "Downloaden", @@ -291,7 +289,6 @@ "enable": "OAuth2-instellingen inschakelen", "domains": "Domeinen", "mobile-apps": "Mobiele applicaties", - "no-mobile-apps": "Geen applicaties geconfigureerd", "mobile-package": "Applicatie pakket", "mobile-package-placeholder": "Vb.: my.example.app", "mobile-package-hint": "Voor Android: uw eigen unieke applicatie-ID. Voor iOS: ID van productbundel.", @@ -299,7 +296,6 @@ "mobile-app-secret": "Geheim van de toepassing", "invalid-mobile-app-secret": "Het geheim van de toepassing mag alleen alfanumerieke tekens bevatten en moet tussen de 16 en 2048 tekens lang zijn.", "copy-mobile-app-secret": "Toepassingsgeheim kopiëren", - "add-mobile-app": "Applicatie toevoegen", "delete-mobile-app": "Toepassingsgegevens verwijderen", "providers": "Providers", "platform-web": "Web", @@ -627,8 +623,6 @@ "filter-type-entity-view-search-query": "Zoekquery voor entiteitsweergave", "filter-type-entity-view-search-query-description": "Entiteitsweergaven met types {{entityViewTypes}} die {{relationType}} relatie hebben {{direction}} {{rootEntity}}", "filter-type-apiUsageState": "Api-gebruiksstatus", - "filter-type-edge-search-query": "Edge-zoekopdracht", - "filter-type-edge-search-query-description": "Edge met types {{edgeTypes}} die {{relationType}} relatie hebben {{direction}} {{rootEntity}}", "entity-filter": "Entiteit filteren", "resolve-multiple": "Oplossen als meerdere entiteiten", "resolve-multiple-hint": "Inschakelen om gegevens van alle gefilterde entiteiten tegelijk weer te geven. \nAls uitgeschakeld, toont de widget alleen gegevens van de geselecteerde entiteit.", @@ -727,7 +721,6 @@ "asset-required": "Asset is vereist", "name-starts-with": "Expressie van itemnaam", "help-text": "Gebruik '%' naar behoefte: '%asset_name_contains%', '%asset_name_ends', 'asset_starts_with'.", - "search": "Assets zoeken", "select-group-to-add": "Selecteer doelgroep om geselecteerde assets toe te voegen", "select-group-to-move": "Selecteer de doelgroep om geselecteerde assets te verplaatsen", "remove-assets-from-group": "Weet je zeker dat je { count, plural, =1 {1 asset} other {# assets} } uit groep '{{entityGroup}}' wilt verwijderen?", @@ -804,7 +797,6 @@ "email-messages": "E-mailberichten", "email-messages-daily-activity": "Dagelijkse activiteit e-mailberichten", "email-messages-monthly-activity": "Maandelijkse activiteit e-mailberichten", - "exceptions": "Uitzonderingen", "executions": "Executies", "javascript": "JavaScript", "javascript-executions": "JavaScript-uitvoeringen", @@ -820,9 +812,7 @@ "permanent-failures": "${entityName} Permanente storingen", "permanent-timeouts": "Permanente time-outs van $ {entityName}", "processing-failures": "${entityName} Verwerkingsfouten", - "processing-failures-and-timeouts": "Verwerkingsfouten en time-outs", "processing-timeouts": "${entityName} time-outs voor verwerking", - "queue-stats": "Queue Statistieken", "rule-chain": "Rule chain", "rule-engine": "Rule engine", "rule-engine-daily-activity": "Dagelijkse activiteit van de rule engine", @@ -969,7 +959,7 @@ "name": "Naam", "name-required": "Naam is verplicht.", "name-max-length": "Naam moet kleiner zijn dan 256 tekens", - "description": "Omschrijving: __________", + "description": "Omschrijving", "decoder": "Decoder", "encoder": "Coderingsprogramma", "test-decoder-fuction": "Test decoder functie", @@ -1047,7 +1037,6 @@ "manage-customer-assets": "Klant devices beheren", "manage-public-assets": "Openbare asset beheren", "manage-customer-edges": "Beheer klant edges", - "manage-public-assets": "Openbare asset beheren", "add-customer-text": "Nieuwe klant toevoegen", "no-customers-text": "Geen klanten gevonden", "customer-details": "Klantgegevens", @@ -2451,7 +2440,7 @@ "name": "Naam", "name-required": "Naam is verplicht.", "name-max-length": "Naam moet kleiner zijn dan 256 tekens", - "description": "Omschrijving: __________", + "description": "Omschrijving", "add": "Entiteitsgroep toevoegen", "open-entity-group": "Entiteitsgroep openen", "add-entity-group-text": "Nieuwe entiteitsgroep toevoegen", @@ -2655,8 +2644,6 @@ "assign-entity-views": "Entiteitsweergaven toewijzen", "assign-entity-views-text": "Wijs { count, plural, =1 {1 entity view} other {# entity views} } toe aan de klant", "delete-entity-views": "Entiteitsweergaven verwijderen", - "make-public": "Entiteitsweergave openbaar maken", - "make-private": "Entiteitsweergave privé maken", "unassign-from-customer": "Toewijzing van klant ongedaan maken", "unassign-entity-views": "Toewijzing van entiteitsweergaven intrekken", "unassign-entity-views-action-title": "Toewijzing { count, plural, =1 {1 entity view} other {# entity views} } van klant ongedaan maken", @@ -2666,10 +2653,6 @@ "delete-entity-views-title": "Weet u zeker dat u { count, plural, =1 {1 entity view} other {# entity views} } wilt verwijderen?", "delete-entity-views-action-title": "Verwijder { count, plural, =1 {1 entity view} other {# entity views} }", "delete-entity-views-text": "Opgelet, na de bevestiging worden alle geselecteerde entiteitsweergaven verwijderd en kunnen alle gerelateerde gegevens niet meer worden hersteld.", - "make-public-entity-view-title": "Weet u zeker dat u de entiteitsweergave '{{entityViewName}}' openbaar wilt maken?", - "make-public-entity-view-text": "Na de bevestiging worden de entiteitsweergave en al haar gegevens openbaar en toegankelijk gemaakt voor anderen.", - "make-private-entity-view-title": "Weet u zeker dat u de entiteitsweergave '{{entityViewName}}' privé wilt maken?", - "make-private-entity-view-text": "Na de bevestiging worden de entiteitsweergave en al zijn gegevens privé gemaakt en zijn ze niet toegankelijk voor anderen.", "unassign-entity-view-title": "Weet u zeker dat u de toewijzing van de entiteitsweergave '{{entityViewName}}' wilt opheffen?", "unassign-entity-view-text": "Na de bevestiging wordt de toewijzing van de entiteitsweergave ongedaan gemaakt en is deze niet toegankelijk voor de klant.", "unassign-entity-view": "Toewijzing van entiteitsweergave ongedaan maken", @@ -2760,7 +2743,6 @@ "type": "Type", "in": "In", "out": "Buiten", - "metadata": "Metagegevens", "message": "Bericht", "entity": "Entiteit", "message-id": "Bericht-ID", @@ -3319,7 +3301,7 @@ "name": "Naam", "name-required": "Naam is verplicht.", "name-max-length": "Naam moet kleiner zijn dan 256 tekens", - "description": "Omschrijving: __________", + "description": "Omschrijving", "base-url": "Basis-URL", "base-url-required": "Basis-URL is vereist", "security-key": "Beveiligingssleutel", @@ -4596,7 +4578,7 @@ "name": "Naam", "name-required": "Naam is verplicht.", "name-max-length": "Naam moet kleiner zijn dan 256 tekens", - "description": "Omschrijving: __________", + "description": "Omschrijving", "events": "Events", "details": "Details", "copyId": "Rol-ID kopiëren", @@ -4641,7 +4623,7 @@ "permissions-required": "Er moet ten minste één machtigingsvermelding worden opgegeven.", "remove-permission": "Machtigingsinvoer verwijderen", "add-permission": "Machtigingsinvoer toevoegen", - "other": "Anders __________", + "other": "Anders", "resource": { "resource": "Hulpbron", "select-resource": "Bron selecteren", @@ -5781,12 +5763,6 @@ "delete-solution-text": "Opgelet, na de bevestiging worden de oplossing en alle gerelateerde gegevens onherstelbaar.", "installing": "Oplossingssjabloon installeren..." }, - "markdown": { - "edit": "Bewerken", - "preview": "Voorbeeld", - "copy-code": "Klik om te kopiëren", - "copied": "Gekopieerd!" - }, "white-labeling": { "white-labeling": "White labelling", "white-labeling-general": "Algemene White Labeling", From 8d749f593b4a14dcb23f959536612ad47c83b65a Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Thu, 27 Mar 2025 13:59:19 +0200 Subject: [PATCH 30/40] MQTT client: limit retransmission attempts to prevent unlimited memory usage and network overload --- .../server/actors/ActorSystemContext.java | 7 +- .../actors/ruleChain/DefaultTbContext.java | 9 +- ...ClientRetransmissionSettingsComponent.java | 37 +++ .../mqtt/MqttClientSettingsComponent.java | 47 ++++ .../src/main/resources/thingsboard.yml | 24 ++ .../server/msa/ContainerTestSuite.java | 12 +- .../msa/connectivity/MqttClientTest.java | 15 +- .../connectivity/MqttGatewayClientTest.java | 14 +- netty-mqtt/pom.xml | 20 ++ .../MaxRetransmissionsReachedException.java | 24 ++ .../thingsboard/mqtt/MqttChannelHandler.java | 30 +-- .../java/org/thingsboard/mqtt/MqttClient.java | 2 +- .../thingsboard/mqtt/MqttClientConfig.java | 20 ++ .../org/thingsboard/mqtt/MqttClientImpl.java | 82 ++++++- .../thingsboard/mqtt/MqttConnectResult.java | 3 + .../thingsboard/mqtt/MqttPendingPublish.java | 134 +++++++---- .../mqtt/MqttPendingSubscription.java | 119 ++++++---- .../mqtt/MqttPendingUnsubscription.java | 85 +++++-- .../org/thingsboard/mqtt/MqttPingHandler.java | 17 +- .../thingsboard/mqtt/PendingOperation.java | 4 +- .../mqtt/RetransmissionHandler.java | 101 +++++++-- .../org/thingsboard/mqtt/MqttClientTest.java | 210 ++++++++++++++++++ .../thingsboard/mqtt/MqttPingHandlerTest.java | 63 ------ .../org/thingsboard/mqtt/MqttTestProxy.java | 202 +++++++++++++++++ .../mqtt/integration/MqttIntegrationTest.java | 151 ------------- .../mqtt/integration/server/MqttServer.java | 84 ------- .../server/MqttTransportHandler.java | 141 ------------ .../test/resources/junit-platform.properties | 3 - pom.xml | 6 + .../rule/engine/api/MqttClientSettings.java | 26 +++ .../rule/engine/api/TbContext.java | 5 + .../rule/engine/mqtt/TbMqttNode.java | 8 + .../rule/engine/mqtt/TbMqttNodeTest.java | 18 ++ 33 files changed, 1100 insertions(+), 623 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientRetransmissionSettingsComponent.java create mode 100644 application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientSettingsComponent.java create mode 100644 netty-mqtt/src/main/java/org/thingsboard/mqtt/MaxRetransmissionsReachedException.java create mode 100644 netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttClientTest.java delete mode 100644 netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttPingHandlerTest.java create mode 100644 netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttTestProxy.java delete mode 100644 netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/MqttIntegrationTest.java delete mode 100644 netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttServer.java delete mode 100644 netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttTransportHandler.java delete mode 100644 netty-mqtt/src/test/resources/junit-platform.properties create mode 100644 rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/MqttClientSettings.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 1ed919e922..78819ab246 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -30,9 +30,10 @@ import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.MailService; +import org.thingsboard.rule.engine.api.MqttClientSettings; import org.thingsboard.rule.engine.api.NotificationCenter; -import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.rule.engine.api.notification.SlackService; import org.thingsboard.rule.engine.api.sms.SmsSenderFactory; @@ -639,6 +640,10 @@ public class ActorSystemContext { @Getter private long cfCalculationResultTimeout; + @Autowired + @Getter + private MqttClientSettings mqttClientSettings; + @Getter @Setter private TbActorSystem actorSystem; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 033e10ca9a..3fb28aee38 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -23,14 +23,15 @@ import org.bouncycastle.util.Arrays; import org.thingsboard.common.util.DebugModeUtil; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ListeningExecutor; +import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.MailService; +import org.thingsboard.rule.engine.api.MqttClientSettings; import org.thingsboard.rule.engine.api.NotificationCenter; import org.thingsboard.rule.engine.api.RuleEngineAlarmService; import org.thingsboard.rule.engine.api.RuleEngineApiUsageStateService; import org.thingsboard.rule.engine.api.RuleEngineAssetProfileCache; import org.thingsboard.rule.engine.api.RuleEngineCalculatedFieldQueueService; import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache; -import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.RuleEngineRpcService; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.ScriptEngine; @@ -1010,13 +1011,17 @@ public class DefaultTbContext implements TbContext { return mainCtx.getAuditLogService(); } + @Override + public MqttClientSettings getMqttClientSettings() { + return mainCtx.getMqttClientSettings(); + } + private TbMsgMetaData getActionMetaData(RuleNodeId ruleNodeId) { TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("ruleNodeId", ruleNodeId.toString()); return metaData; } - @Override public void schedule(Runnable runnable, long delay, TimeUnit timeUnit) { mainCtx.getScheduler().schedule(runnable, delay, timeUnit); diff --git a/application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientRetransmissionSettingsComponent.java b/application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientRetransmissionSettingsComponent.java new file mode 100644 index 0000000000..33e9358d2b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientRetransmissionSettingsComponent.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.config.mqtt; + +import jakarta.validation.constraints.PositiveOrZero; +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; +import org.springframework.validation.annotation.Validated; + +@Data +@Validated +@Configuration +@ConfigurationProperties(prefix = "mqtt.client.retransmission") +public class MqttClientRetransmissionSettingsComponent { + + @PositiveOrZero + private int maxAttempts; + @PositiveOrZero + private long initialDelayMillis; + @PositiveOrZero + private double jitterFactor; + +} diff --git a/application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientSettingsComponent.java b/application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientSettingsComponent.java new file mode 100644 index 0000000000..25df212925 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientSettingsComponent.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.config.mqtt; + +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import org.springframework.context.annotation.Configuration; +import org.thingsboard.rule.engine.api.MqttClientSettings; + +@ToString +@EqualsAndHashCode +@Configuration +@RequiredArgsConstructor +public class MqttClientSettingsComponent implements MqttClientSettings { + + private final MqttClientRetransmissionSettingsComponent retransmissionSettingsComponent; + + @Override + public int getRetransmissionMaxAttempts() { + return retransmissionSettingsComponent.getMaxAttempts(); + } + + @Override + public long getRetransmissionInitialDelayMillis() { + return retransmissionSettingsComponent.getInitialDelayMillis(); + } + + @Override + public double getRetransmissionJitterFactor() { + return retransmissionSettingsComponent.getJitterFactor(); + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 721d23b700..d7a55e496e 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1920,3 +1920,27 @@ mobileApp: googlePlayLink: "${TB_MOBILE_APP_GOOGLE_PLAY_LINK:https://play.google.com/store/apps/details?id=org.thingsboard.demo.app}" # Link to App Store for Thingsboard Live mobile application appStoreLink: "${TB_MOBILE_APP_APP_STORE_LINK:https://apps.apple.com/us/app/thingsboard-live/id1594355695}" + +mqtt: + # MQTT client configuration parameters + client: + # Parameters that control the retransmission mechanism. + # This mechanism only applies to the handling of MQTT Publish, Subscribe, Unsubscribe and Pubrel messages. + # With the updated default settings: + # - After sending the message, wait approximately 5000 ms (± jitter) for the 1st attempt. + # - The 2nd attempt will occur after roughly 5000 * 2 = 10,000 ms (± jitter). + # - The 3rd attempt will occur after roughly 5000 * 4 = 20,000 ms (± jitter). + # - The 4th "attempt" will not actually perform a retransmission. + # Instead, the system will detect that the maximum number of attempts has been reached and drop the pending message. + retransmission: + # Maximum number of retransmission attempts allowed. + # If the attempt count exceeds this value, retransmissions will stop and the pending message will be dropped. + max_attempts: "${TB_MQTT_CLIENT_RETRANSMISSION_MAX_ATTEMPTS:3}" + # Base delay (in milliseconds) before the first retransmission attempt, measured from the moment the message is sent. + # Subsequent delays are calculated using exponential backoff. + # This base delay is also used as the reference value for applying jitter. + initial_delay_millis: "${TB_MQTT_CLIENT_RETRANSMISSION_INITIAL_DELAY_MILLIS:5000}" + # Jitter factor applied to the calculated retransmission delay. + # The actual delay is randomized within a range defined by multiplying the base delay by a factor between (1 - jitter_factor) and (1 + jitter_factor). + # For example, a jitter_factor of 0.15 means the actual delay may vary by up to ±15% of the base delay. + jitter_factor: "${TB_MQTT_CLIENT_RETRANSMISSION_JITTER_FACTOR:0.15}" diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 65def6a964..9b9bb31dc1 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -55,8 +55,8 @@ public class ContainerTestSuite { private static final String TB_JS_EXECUTOR_LOG_REGEXP = ".*template started.*"; private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(400); - private DockerComposeContainer testContainer; - private ThingsBoardDbInstaller installTb; + private DockerComposeContainer testContainer; + private ThingsBoardDbInstaller installTb; private boolean isActive; private static ContainerTestSuite containerTestSuite; @@ -194,7 +194,7 @@ public class ContainerTestSuite { setActive(true); } catch (Exception e) { log.error("Failed to create test container", e); - fail("Failed to create test container"); + fail("Failed to create test container", e); } } @@ -263,7 +263,7 @@ public class ContainerTestSuite { log.info("Trying to delete temp dir {}", targetDir); FileUtils.deleteDirectory(new File(targetDir)); } catch (IOException e) { - log.error("Can't delete temp directory " + targetDir, e); + log.error("Can't delete temp directory {}", targetDir, e); } } @@ -286,8 +286,8 @@ public class ContainerTestSuite { FileUtils.writeStringToFile(file, outputContent, StandardCharsets.UTF_8); assertThat(FileUtils.readFileToString(file, StandardCharsets.UTF_8), is(outputContent)); } catch (IOException e) { - log.error("failed to update file " + sourceFilename, e); - fail("failed to update file"); + log.error("failed to update file {}", sourceFilename, e); + fail("failed to update file", e); } } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java index ebdfb4e3c9..dacfea9b10 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java @@ -42,7 +42,6 @@ import org.thingsboard.mqtt.MqttClient; import org.thingsboard.mqtt.MqttClientCallback; import org.thingsboard.mqtt.MqttClientConfig; import org.thingsboard.mqtt.MqttHandler; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileProvisionType; @@ -82,7 +81,6 @@ import java.util.concurrent.TimeoutException; import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.fail; -import static org.thingsboard.server.common.data.DataConstants.DEVICE; import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultDevicePrototype; @@ -301,7 +299,7 @@ public class MqttClientTest extends AbstractContainerTest { assertThat(Objects.requireNonNull(requestFromServer).getMessage()).isEqualTo("{\"method\":\"getValue\",\"params\":true}"); - Integer requestId = Integer.valueOf(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length())); + int requestId = Integer.parseInt(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length())); JsonObject clientResponse = new JsonObject(); clientResponse.addProperty("response", "someResponse"); // Send a response to the server's RPC request @@ -340,7 +338,7 @@ public class MqttClientTest extends AbstractContainerTest { assertThat(Objects.requireNonNull(requestFromServer).getMessage()).isEqualTo("{\"method\":\"getValue\",\"params\":true}"); - Integer requestId = Integer.valueOf(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length())); + int requestId = Integer.parseInt(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length())); JsonObject clientResponse = new JsonObject(); clientResponse.addProperty("response", "someResponse"); // Send a response to the server's RPC request @@ -520,13 +518,13 @@ public class MqttClientTest extends AbstractContainerTest { mqttClient.on("/provision/response", listener, MqttQoS.AT_LEAST_ONCE).get(3 * timeoutMultiplier, TimeUnit.SECONDS); TimeUnit.SECONDS.sleep(2 * timeoutMultiplier); assertThat(subAckResult[0]).isNotNull(); - assertThat(MqttReasonCodes.SubAck.GRANTED_QOS_1.equals(subAckResult[0])); + assertThat(MqttReasonCodes.SubAck.GRANTED_QOS_1).isEqualTo(subAckResult[0]); subAckResult[0] = null; mqttClient.on("v1/devices/me/attributes", listener, MqttQoS.AT_LEAST_ONCE).get(3 * timeoutMultiplier, TimeUnit.SECONDS); TimeUnit.SECONDS.sleep(2 * timeoutMultiplier); assertThat(subAckResult[0]).isNotNull(); - assertThat(MqttReasonCodes.SubAck.TOPIC_FILTER_INVALID.equals(subAckResult[0])); + assertThat(MqttReasonCodes.SubAck.TOPIC_FILTER_INVALID).isEqualTo(subAckResult[0]); testRestClient.deleteDeviceIfExists(device.getId()); updateDeviceProfileWithProvisioningStrategy(deviceProfile, DeviceProfileProvisionType.DISABLED); @@ -596,7 +594,7 @@ public class MqttClientTest extends AbstractContainerTest { .await() .alias("Check device disconnect.") .atMost(TIMEOUT*timeoutMultiplier, TimeUnit.SECONDS) - .until(() -> returnCodeByteValue.size() > 0); + .until(() -> !returnCodeByteValue.isEmpty()); assertThat(returnCodeByteValueSecondClient).isEmpty(); assertThat(returnCodeByteValue).isNotEmpty(); @@ -663,7 +661,7 @@ public class MqttClientTest extends AbstractContainerTest { .stream() .filter(RuleChain::isRoot) .findFirst(); - if (!defaultRuleChain.isPresent()) { + if (defaultRuleChain.isEmpty()) { fail("Root rule chain wasn't found"); } return defaultRuleChain.get().getId(); @@ -717,6 +715,7 @@ public class MqttClientTest extends AbstractContainerTest { clientConfig.setClientId("MQTT client from test"); clientConfig.setUsername(username); clientConfig.setProtocolVersion(mqttVersion); + clientConfig.setRetransmissionConfig(new MqttClientConfig.RetransmissionConfig(5, 5000L, 0.1d)); // same as defaults in thingsboard.yml as of time of this writing MqttClient mqttClient = MqttClient.create(clientConfig, listener, handlerExecutor); if (connect) { mqttClient.connect(TRANSPORT_HOST, TRANSPORT_PORT).get(); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java index cc587fbbd5..32e8498f45 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java @@ -39,7 +39,6 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.mqtt.MqttClient; import org.thingsboard.mqtt.MqttClientConfig; import org.thingsboard.mqtt.MqttHandler; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceId; @@ -65,7 +64,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; -import static org.thingsboard.server.common.data.DataConstants.DEVICE; import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultGatewayPrototype; @@ -76,7 +74,6 @@ public class MqttGatewayClientTest extends AbstractContainerTest { private MqttClient mqttClient; private Device createdDevice; private MqttMessageListener listener; - private JsonParser jsonParser = new JsonParser(); AbstractListeningExecutor handlerExecutor; @@ -100,7 +97,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest { } @AfterMethod - public void removeGateway() { + public void removeGateway() { testRestClient.deleteDeviceIfExists(this.gatewayDevice.getId()); testRestClient.deleteDeviceIfExists(this.createdDevice.getId()); this.listener = null; @@ -197,7 +194,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest { mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(requestData.toString().getBytes())).get(); event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - JsonObject responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); + JsonObject responseData = JsonParser.parseString(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); assertThat(responseData.has("value")).isTrue(); assertThat(responseData.get("value").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString()); @@ -213,7 +210,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest { mqttClient.on("v1/gateway/attributes/response", listener, MqttQoS.AT_LEAST_ONCE).get(); mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(requestData.toString().getBytes())).get(); event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); + responseData = JsonParser.parseString(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); assertThat(responseData.has("values")).isTrue(); assertThat(responseData.get("values").getAsJsonObject().get("attr1").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString()); @@ -231,7 +228,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest { mqttClient.on("v1/gateway/attributes/response", listener, MqttQoS.AT_LEAST_ONCE).get(); mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(requestData.toString().getBytes())).get(); event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); + responseData = JsonParser.parseString(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); assertThat(responseData.has("values")).isTrue(); assertThat(responseData.get("values").getAsJsonObject().get("attr1").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString()); @@ -390,7 +387,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest { mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(gatewayAttributesRequest.toString().getBytes())).get(); MqttEvent clientAttributeEvent = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); assertThat(clientAttributeEvent).isNotNull(); - JsonObject responseMessage = new JsonParser().parse(Objects.requireNonNull(clientAttributeEvent).getMessage()).getAsJsonObject(); + JsonObject responseMessage = JsonParser.parseString(Objects.requireNonNull(clientAttributeEvent).getMessage()).getAsJsonObject(); assertThat(responseMessage.get("id").getAsInt()).isEqualTo(messageId); assertThat(responseMessage.get("device").getAsString()).isEqualTo(createdDevice.getName()); @@ -427,6 +424,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest { clientConfig.setOwnerId(getOwnerId()); clientConfig.setClientId("MQTT client from test"); clientConfig.setUsername(deviceCredentials.getCredentialsId()); + clientConfig.setRetransmissionConfig(new MqttClientConfig.RetransmissionConfig(3, 5000L, 0.1d)); // same as defaults in thingsboard.yml as of time of this writing MqttClient mqttClient = MqttClient.create(clientConfig, listener, handlerExecutor); mqttClient.connect("localhost", 1883).get(); return mqttClient; diff --git a/netty-mqtt/pom.xml b/netty-mqtt/pom.xml index b5fd83a54f..f9aa80c78e 100644 --- a/netty-mqtt/pom.xml +++ b/netty-mqtt/pom.xml @@ -87,6 +87,26 @@ awaitility test + + org.testcontainers + testcontainers + test + + + org.testcontainers + junit-jupiter + test + + + software.xdev + testcontainers-junit4-mock + test + + + org.testcontainers + hivemq + test + diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MaxRetransmissionsReachedException.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MaxRetransmissionsReachedException.java new file mode 100644 index 0000000000..3d483dd541 --- /dev/null +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MaxRetransmissionsReachedException.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt; + +public class MaxRetransmissionsReachedException extends RuntimeException { + + public MaxRetransmissionsReachedException(String message) { + super(message); + } + +} diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttChannelHandler.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttChannelHandler.java index ad976c848a..9686a2b1d7 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttChannelHandler.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttChannelHandler.java @@ -57,7 +57,7 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler } @Override - protected void channelRead0(ChannelHandlerContext ctx, MqttMessage msg) throws Exception { + protected void channelRead0(ChannelHandlerContext ctx, MqttMessage msg) { if (msg.decoderResult().isSuccess()) { switch (msg.fixedHeader().messageType()) { case CONNACK: @@ -120,6 +120,7 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler this.client.getClientConfig().getUsername(), this.client.getClientConfig().getPassword() != null ? this.client.getClientConfig().getPassword().getBytes(CharsetUtil.UTF_8) : null ); + log.debug("{} Sending CONNECT", client.getClientConfig().getOwnerId()); ctx.channel().writeAndFlush(new MqttConnectMessage(fixedHeader, variableHeader, payload)); } @@ -173,6 +174,7 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler } private void handleConack(Channel channel, MqttConnAckMessage message) { + log.debug("{} Handling CONNACK", client.getClientConfig().getOwnerId()); switch (message.variableHeader().connectReturnCode()) { case CONNECTION_ACCEPTED: this.connectFuture.setSuccess(new MqttConnectResult(true, MqttConnectReturnCode.CONNECTION_ACCEPTED, channel.closeFuture())); @@ -219,9 +221,9 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler } pendingSubscription.onSubackReceived(); for (MqttPendingSubscription.MqttPendingHandler handler : pendingSubscription.getHandlers()) { - MqttSubscription subscription = new MqttSubscription(pendingSubscription.getTopic(), handler.getHandler(), handler.isOnce()); + MqttSubscription subscription = new MqttSubscription(pendingSubscription.getTopic(), handler.handler(), handler.once()); this.client.getSubscriptions().put(pendingSubscription.getTopic(), subscription); - this.client.getHandlerToSubscription().put(handler.getHandler(), subscription); + this.client.getHandlerToSubscription().put(handler.handler(), subscription); } this.client.getPendingSubscribeTopics().remove(pendingSubscription.getTopic()); @@ -282,17 +284,16 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler } private void handlePuback(MqttPubAckMessage message) { - MqttPendingPublish pendingPublish = this.client.getPendingPublishes().get(message.variableHeader().messageId()); - if (pendingPublish == null) { - return; - } - pendingPublish.getFuture().setSuccess(null); - pendingPublish.onPubackReceived(); - this.client.getPendingPublishes().remove(message.variableHeader().messageId()); - pendingPublish.getPayload().release(); - if (this.client.getCallback() != null) { - this.client.getCallback().onPubAck(message); - } + log.trace("{} Handling PUBACK", client.getClientConfig().getOwnerId()); + client.getPendingPublishes().computeIfPresent(message.variableHeader().messageId(), (__, pendingPublish) -> { + pendingPublish.getFuture().setSuccess(null); + pendingPublish.onPubackReceived(); + pendingPublish.getPayload().release(); + if (client.getCallback() != null) { + client.getCallback().onPubAck(message); + } + return null; + }); } private void handlePubrec(Channel channel, MqttMessage message) { @@ -335,6 +336,7 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler } private void handleDisconnect(MqttMessage message) { + log.debug("{} Handling DISCONNECT", client.getClientConfig().getOwnerId()); if (this.client.getCallback() != null) { this.client.getCallback().onDisconnect(message); } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClient.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClient.java index db0459e08a..4d845320e8 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClient.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClient.java @@ -184,7 +184,7 @@ public interface MqttClient { * @param config The config object to use while looking for settings * @param defaultHandler The handler for incoming messages that do not match any topic subscriptions */ - static MqttClient create(MqttClientConfig config, MqttHandler defaultHandler, ListeningExecutor handlerExecutor){ + static MqttClient create(MqttClientConfig config, MqttHandler defaultHandler, ListeningExecutor handlerExecutor) { return new MqttClientImpl(config, defaultHandler, handlerExecutor); } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientConfig.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientConfig.java index 41df077d71..24feb3e58e 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientConfig.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientConfig.java @@ -47,6 +47,26 @@ public final class MqttClientConfig { private long reconnectDelay = 1L; private int maxBytesInMessage = 8092; + @Getter + @Setter + private RetransmissionConfig retransmissionConfig; + + public record RetransmissionConfig(int maxAttempts, long initialDelayMillis, double jitterFactor) { + + public RetransmissionConfig { + if (maxAttempts < 0) { + throw new IllegalArgumentException("Max retransmission attempts (maxAttempts) must be zero or greater, but was " + maxAttempts); + } + if (initialDelayMillis < 0) { + throw new IllegalArgumentException("Initial retransmission delay (initialDelayMillis) must be zero or greater, but was " + initialDelayMillis); + } + if (jitterFactor < 0) { + throw new IllegalArgumentException("Jitter factor (jitterFactor) must be zero or greater, but was " + jitterFactor); + } + } + + } + public MqttClientConfig() { this(null); } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java index 47eae565dc..ee07752db3 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java @@ -17,6 +17,7 @@ package org.thingsboard.mqtt; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import io.netty.bootstrap.Bootstrap; import io.netty.buffer.ByteBuf; import io.netty.channel.Channel; @@ -384,8 +385,33 @@ final class MqttClientImpl implements MqttClient { MqttFixedHeader fixedHeader = new MqttFixedHeader(MqttMessageType.PUBLISH, false, qos, retain, 0); MqttPublishVariableHeader variableHeader = new MqttPublishVariableHeader(topic, getNewMessageId().messageId()); MqttPublishMessage message = new MqttPublishMessage(fixedHeader, variableHeader, payload); - MqttPendingPublish pendingPublish = new MqttPendingPublish(variableHeader.packetId(), future, - payload.retain(), message, qos, () -> !pendingPublishes.containsKey(variableHeader.packetId())); + + final var pendingPublish = MqttPendingPublish.builder() + .messageId(variableHeader.packetId()) + .future(future) + .payload(payload.retain()) + .message(message) + .qos(qos) + .ownerId(clientConfig.getOwnerId()) + .retransmissionConfig(clientConfig.getRetransmissionConfig()) + .pendingOperation(new PendingOperation() { + @Override + public boolean isCancelled() { + return !pendingPublishes.containsKey(variableHeader.packetId()); + } + + @Override + public void onMaxRetransmissionAttemptsReached() { + pendingPublishes.computeIfPresent(variableHeader.packetId(), (__, pendingPublish) -> { + var message = "Unable to deliver publish message due to max retransmission attempts (%s) being reached for client '%s' on topic '%s' (message ID: %d)" + .formatted(clientConfig.getRetransmissionConfig().maxAttempts(), clientConfig.getClientId(), topic, variableHeader.packetId()); + pendingPublish.getFuture().tryFailure(new MaxRetransmissionsReachedException(message)); + pendingPublish.getPayload().release(); + return null; + }); + } + }).build(); + this.pendingPublishes.put(pendingPublish.getMessageId(), pendingPublish); ChannelFuture channelFuture = this.sendAndFlushPacket(message); @@ -499,9 +525,30 @@ final class MqttClientImpl implements MqttClient { MqttSubscribePayload payload = new MqttSubscribePayload(Collections.singletonList(subscription)); MqttSubscribeMessage message = new MqttSubscribeMessage(fixedHeader, variableHeader, payload); - final MqttPendingSubscription pendingSubscription = new MqttPendingSubscription(future, topic, message, - () -> !pendingSubscriptions.containsKey(variableHeader.messageId())); - pendingSubscription.addHandler(handler, once); + final var pendingSubscription = MqttPendingSubscription.builder() + .future(future) + .topic(topic) + .handlers(Sets.newHashSet(new MqttPendingSubscription.MqttPendingHandler(handler, once))) + .subscribeMessage(message) + .ownerId(clientConfig.getOwnerId()) + .retransmissionConfig(clientConfig.getRetransmissionConfig()) + .pendingOperation(new PendingOperation() { + @Override + public boolean isCancelled() { + return !pendingSubscriptions.containsKey(variableHeader.messageId()); + } + + @Override + public void onMaxRetransmissionAttemptsReached() { + pendingSubscriptions.computeIfPresent(variableHeader.messageId(), (__, pendingSubscription) -> { + var message = "Unable to deliver subscribe message due to max retransmission attempts (%s) being reached for client '%s' on topic '%s' (message ID: %d)" + .formatted(clientConfig.getRetransmissionConfig().maxAttempts(), clientConfig.getClientId(), topic, variableHeader.messageId()); + pendingSubscription.getFuture().tryFailure(new MaxRetransmissionsReachedException(message)); + return null; + }); + } + }).build(); + this.pendingSubscriptions.put(variableHeader.messageId(), pendingSubscription); this.pendingSubscribeTopics.add(topic); pendingSubscription.setSent(this.sendAndFlushPacket(message) != null); //If not sent, we will send it when the connection is opened @@ -518,8 +565,29 @@ final class MqttClientImpl implements MqttClient { MqttUnsubscribePayload payload = new MqttUnsubscribePayload(Collections.singletonList(topic)); MqttUnsubscribeMessage message = new MqttUnsubscribeMessage(fixedHeader, variableHeader, payload); - MqttPendingUnsubscription pendingUnsubscription = new MqttPendingUnsubscription(promise, topic, message, - () -> !pendingServerUnsubscribes.containsKey(variableHeader.messageId())); + final var pendingUnsubscription = MqttPendingUnsubscription.builder() + .future(promise) + .topic(topic) + .unsubscribeMessage(message) + .ownerId(clientConfig.getOwnerId()) + .retransmissionConfig(clientConfig.getRetransmissionConfig()) + .pendingOperation(new PendingOperation() { + @Override + public boolean isCancelled() { + return !pendingServerUnsubscribes.containsKey(variableHeader.messageId()); + } + + @Override + public void onMaxRetransmissionAttemptsReached() { + pendingServerUnsubscribes.computeIfPresent(variableHeader.messageId(), (__, pendingUnsubscription) -> { + var message = "Unable to deliver unsubscribe message due to max retransmission attempts (%s) being reached for client '%s' on topic '%s' (message ID: %d)" + .formatted(clientConfig.getRetransmissionConfig().maxAttempts(), clientConfig.getClientId(), topic, variableHeader.messageId()); + pendingUnsubscription.getFuture().tryFailure(new MaxRetransmissionsReachedException(message)); + return null; + }); + } + }).build(); + this.pendingServerUnsubscribes.put(variableHeader.messageId(), pendingUnsubscription); pendingUnsubscription.startRetransmissionTimer(this.eventLoop.next(), this::sendAndFlushPacket); diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttConnectResult.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttConnectResult.java index 911bc1d395..67757d2a7a 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttConnectResult.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttConnectResult.java @@ -17,7 +17,9 @@ package org.thingsboard.mqtt; import io.netty.channel.ChannelFuture; import io.netty.handler.codec.mqtt.MqttConnectReturnCode; +import lombok.ToString; +@ToString @SuppressWarnings({"WeakerAccess", "unused"}) public final class MqttConnectResult { @@ -42,4 +44,5 @@ public final class MqttConnectResult { public ChannelFuture getCloseFuture() { return closeFuture; } + } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingPublish.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingPublish.java index e8c3ef35f7..1846bdb12b 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingPublish.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingPublish.java @@ -21,9 +21,13 @@ import io.netty.handler.codec.mqtt.MqttMessage; import io.netty.handler.codec.mqtt.MqttPublishMessage; import io.netty.handler.codec.mqtt.MqttQoS; import io.netty.util.concurrent.Promise; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.Setter; import java.util.function.Consumer; +@Getter(AccessLevel.PACKAGE) final class MqttPendingPublish { private final int messageId; @@ -32,80 +36,126 @@ final class MqttPendingPublish { private final MqttPublishMessage message; private final MqttQoS qos; + @Getter(AccessLevel.NONE) private final RetransmissionHandler publishRetransmissionHandler; + @Getter(AccessLevel.NONE) private final RetransmissionHandler pubrelRetransmissionHandler; + @Setter(AccessLevel.PACKAGE) private boolean sent = false; - MqttPendingPublish(int messageId, Promise future, ByteBuf payload, MqttPublishMessage message, MqttQoS qos, PendingOperation operation) { + private MqttPendingPublish( + int messageId, + Promise future, + ByteBuf payload, + MqttPublishMessage message, + MqttQoS qos, + String ownerId, + MqttClientConfig.RetransmissionConfig retransmissionConfig, + PendingOperation pendingOperation + ) { this.messageId = messageId; this.future = future; this.payload = payload; this.message = message; this.qos = qos; - this.publishRetransmissionHandler = new RetransmissionHandler<>(operation); - this.publishRetransmissionHandler.setOriginalMessage(message); - this.pubrelRetransmissionHandler = new RetransmissionHandler<>(operation); - } - - int getMessageId() { - return messageId; - } - - Promise getFuture() { - return future; - } - - ByteBuf getPayload() { - return payload; - } - - boolean isSent() { - return sent; - } - - void setSent(boolean sent) { - this.sent = sent; - } - - MqttPublishMessage getMessage() { - return message; - } - - MqttQoS getQos() { - return qos; + publishRetransmissionHandler = new RetransmissionHandler<>(retransmissionConfig, pendingOperation, ownerId); + publishRetransmissionHandler.setOriginalMessage(message); + pubrelRetransmissionHandler = new RetransmissionHandler<>(retransmissionConfig, pendingOperation, ownerId); } void startPublishRetransmissionTimer(EventLoop eventLoop, Consumer sendPacket) { - this.publishRetransmissionHandler.setHandle(((fixedHeader, originalMessage) -> - sendPacket.accept(new MqttPublishMessage(fixedHeader, originalMessage.variableHeader(), this.payload.retain())))); - this.publishRetransmissionHandler.start(eventLoop); + publishRetransmissionHandler.setHandler(((fixedHeader, originalMessage) -> + sendPacket.accept(new MqttPublishMessage(fixedHeader, originalMessage.variableHeader(), payload.retain())))); + publishRetransmissionHandler.start(eventLoop); } void onPubackReceived() { - this.publishRetransmissionHandler.stop(); + publishRetransmissionHandler.stop(); } void setPubrelMessage(MqttMessage pubrelMessage) { - this.pubrelRetransmissionHandler.setOriginalMessage(pubrelMessage); + pubrelRetransmissionHandler.setOriginalMessage(pubrelMessage); } void startPubrelRetransmissionTimer(EventLoop eventLoop, Consumer sendPacket) { - this.pubrelRetransmissionHandler.setHandle((fixedHeader, originalMessage) -> + pubrelRetransmissionHandler.setHandler((fixedHeader, originalMessage) -> sendPacket.accept(new MqttMessage(fixedHeader, originalMessage.variableHeader()))); - this.pubrelRetransmissionHandler.start(eventLoop); + pubrelRetransmissionHandler.start(eventLoop); } void onPubcompReceived() { - this.pubrelRetransmissionHandler.stop(); + pubrelRetransmissionHandler.stop(); } void onChannelClosed() { - this.publishRetransmissionHandler.stop(); - this.pubrelRetransmissionHandler.stop(); + publishRetransmissionHandler.stop(); + pubrelRetransmissionHandler.stop(); if (payload != null) { payload.release(); } } + + static Builder builder() { + return new Builder(); + } + + static class Builder { + + private int messageId; + private Promise future; + private ByteBuf payload; + private MqttPublishMessage message; + private MqttQoS qos; + private String ownerId; + private MqttClientConfig.RetransmissionConfig retransmissionConfig; + private PendingOperation pendingOperation; + + Builder messageId(int messageId) { + this.messageId = messageId; + return this; + } + + Builder future(Promise future) { + this.future = future; + return this; + } + + Builder payload(ByteBuf payload) { + this.payload = payload; + return this; + } + + Builder message(MqttPublishMessage message) { + this.message = message; + return this; + } + + Builder qos(MqttQoS qos) { + this.qos = qos; + return this; + } + + Builder ownerId(String ownerId) { + this.ownerId = ownerId; + return this; + } + + Builder retransmissionConfig(MqttClientConfig.RetransmissionConfig retransmissionConfig) { + this.retransmissionConfig = retransmissionConfig; + return this; + } + + Builder pendingOperation(PendingOperation pendingOperation) { + this.pendingOperation = pendingOperation; + return this; + } + + MqttPendingPublish build() { + return new MqttPendingPublish(messageId, future, payload, message, qos, ownerId, retransmissionConfig, pendingOperation); + } + + } + } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingSubscription.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingSubscription.java index af5d53a06c..7b2ba613cb 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingSubscription.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingSubscription.java @@ -18,90 +18,123 @@ package org.thingsboard.mqtt; import io.netty.channel.EventLoop; import io.netty.handler.codec.mqtt.MqttSubscribeMessage; import io.netty.util.concurrent.Promise; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.Setter; import java.util.HashSet; import java.util.Set; import java.util.function.Consumer; +import static java.util.Objects.requireNonNullElseGet; + +@Getter(AccessLevel.PACKAGE) final class MqttPendingSubscription { private final Promise future; private final String topic; - private final Set handlers = new HashSet<>(); + private final Set handlers; private final MqttSubscribeMessage subscribeMessage; + @Getter(AccessLevel.NONE) private final RetransmissionHandler retransmissionHandler; + @Setter(AccessLevel.PACKAGE) private boolean sent = false; - MqttPendingSubscription(Promise future, String topic, MqttSubscribeMessage message, PendingOperation operation) { + private MqttPendingSubscription( + Promise future, + String topic, + Set handlers, + MqttSubscribeMessage subscribeMessage, + String ownerId, + MqttClientConfig.RetransmissionConfig retransmissionConfig, + PendingOperation operation + ) { this.future = future; this.topic = topic; - this.subscribeMessage = message; + this.handlers = requireNonNullElseGet(handlers, HashSet::new); + this.subscribeMessage = subscribeMessage; - this.retransmissionHandler = new RetransmissionHandler<>(operation); - this.retransmissionHandler.setOriginalMessage(message); + retransmissionHandler = new RetransmissionHandler<>(retransmissionConfig, operation, ownerId); + retransmissionHandler.setOriginalMessage(subscribeMessage); } - Promise getFuture() { - return future; - } + record MqttPendingHandler(MqttHandler handler, boolean once) {} - String getTopic() { - return topic; + void addHandler(MqttHandler handler, boolean once) { + handlers.add(new MqttPendingHandler(handler, once)); } - boolean isSent() { - return sent; + void startRetransmitTimer(EventLoop eventLoop, Consumer sendPacket) { + if (sent) { // If the packet is sent, we can start the retransmission timer + retransmissionHandler.setHandler((fixedHeader, originalMessage) -> + sendPacket.accept(new MqttSubscribeMessage(fixedHeader, originalMessage.variableHeader(), originalMessage.payload()))); + retransmissionHandler.start(eventLoop); + } } - void setSent(boolean sent) { - this.sent = sent; + void onSubackReceived() { + retransmissionHandler.stop(); } - MqttSubscribeMessage getSubscribeMessage() { - return subscribeMessage; + void onChannelClosed() { + retransmissionHandler.stop(); } - void addHandler(MqttHandler handler, boolean once) { - this.handlers.add(new MqttPendingHandler(handler, once)); + static Builder builder() { + return new Builder(); } - Set getHandlers() { - return handlers; - } + static class Builder { - void startRetransmitTimer(EventLoop eventLoop, Consumer sendPacket) { - if (this.sent) { //If the packet is sent, we can start the retransmit timer - this.retransmissionHandler.setHandle((fixedHeader, originalMessage) -> - sendPacket.accept(new MqttSubscribeMessage(fixedHeader, originalMessage.variableHeader(), originalMessage.payload()))); - this.retransmissionHandler.start(eventLoop); + private Promise future; + private String topic; + private Set handlers; + private MqttSubscribeMessage subscribeMessage; + private String ownerId; + private PendingOperation pendingOperation; + private MqttClientConfig.RetransmissionConfig retransmissionConfig; + + Builder future(Promise future) { + this.future = future; + return this; } - } - void onSubackReceived() { - this.retransmissionHandler.stop(); - } + Builder topic(String topic) { + this.topic = topic; + return this; + } - final class MqttPendingHandler { - private final MqttHandler handler; - private final boolean once; + Builder handlers(Set handlers) { + this.handlers = handlers; + return this; + } - MqttPendingHandler(MqttHandler handler, boolean once) { - this.handler = handler; - this.once = once; + Builder subscribeMessage(MqttSubscribeMessage subscribeMessage) { + this.subscribeMessage = subscribeMessage; + return this; } - MqttHandler getHandler() { - return handler; + Builder ownerId(String ownerId) { + this.ownerId = ownerId; + return this; } - boolean isOnce() { - return once; + Builder retransmissionConfig(MqttClientConfig.RetransmissionConfig retransmissionConfig) { + this.retransmissionConfig = retransmissionConfig; + return this; + } + + Builder pendingOperation(PendingOperation pendingOperation) { + this.pendingOperation = pendingOperation; + return this; + } + + MqttPendingSubscription build() { + return new MqttPendingSubscription(future, topic, handlers, subscribeMessage, ownerId, retransmissionConfig, pendingOperation); } - } - void onChannelClosed() { - this.retransmissionHandler.stop(); } + } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingUnsubscription.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingUnsubscription.java index 9cb3bd2f8d..8bc23292f8 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingUnsubscription.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingUnsubscription.java @@ -18,43 +18,96 @@ package org.thingsboard.mqtt; import io.netty.channel.EventLoop; import io.netty.handler.codec.mqtt.MqttUnsubscribeMessage; import io.netty.util.concurrent.Promise; +import lombok.AccessLevel; +import lombok.Getter; import java.util.function.Consumer; -final class MqttPendingUnsubscription{ +@Getter(AccessLevel.PACKAGE) +final class MqttPendingUnsubscription { private final Promise future; private final String topic; + @Getter(AccessLevel.NONE) private final RetransmissionHandler retransmissionHandler; - MqttPendingUnsubscription(Promise future, String topic, MqttUnsubscribeMessage unsubscribeMessage, PendingOperation operation) { + private MqttPendingUnsubscription( + Promise future, + String topic, + MqttUnsubscribeMessage unsubscribeMessage, + String ownerId, + MqttClientConfig.RetransmissionConfig retransmissionConfig, + PendingOperation operation + ) { this.future = future; this.topic = topic; - this.retransmissionHandler = new RetransmissionHandler<>(operation); - this.retransmissionHandler.setOriginalMessage(unsubscribeMessage); + retransmissionHandler = new RetransmissionHandler<>(retransmissionConfig, operation, ownerId); + retransmissionHandler.setOriginalMessage(unsubscribeMessage); } - Promise getFuture() { - return future; + void startRetransmissionTimer(EventLoop eventLoop, Consumer sendPacket) { + retransmissionHandler.setHandler((fixedHeader, originalMessage) -> + sendPacket.accept(new MqttUnsubscribeMessage(fixedHeader, originalMessage.variableHeader(), originalMessage.payload()))); + retransmissionHandler.start(eventLoop); } - String getTopic() { - return topic; + void onUnsubackReceived() { + retransmissionHandler.stop(); } - void startRetransmissionTimer(EventLoop eventLoop, Consumer sendPacket) { - this.retransmissionHandler.setHandle((fixedHeader, originalMessage) -> - sendPacket.accept(new MqttUnsubscribeMessage(fixedHeader, originalMessage.variableHeader(), originalMessage.payload()))); - this.retransmissionHandler.start(eventLoop); + void onChannelClosed() { + retransmissionHandler.stop(); } - void onUnsubackReceived(){ - this.retransmissionHandler.stop(); + static Builder builder() { + return new Builder(); } - void onChannelClosed(){ - this.retransmissionHandler.stop(); + static class Builder { + + private Promise future; + private String topic; + private MqttUnsubscribeMessage unsubscribeMessage; + private String ownerId; + private PendingOperation pendingOperation; + private MqttClientConfig.RetransmissionConfig retransmissionConfig; + + Builder future(Promise future) { + this.future = future; + return this; + } + + Builder topic(String topic) { + this.topic = topic; + return this; + } + + Builder unsubscribeMessage(MqttUnsubscribeMessage unsubscribeMessage) { + this.unsubscribeMessage = unsubscribeMessage; + return this; + } + + Builder ownerId(String ownerId) { + this.ownerId = ownerId; + return this; + } + + Builder retransmissionConfig(MqttClientConfig.RetransmissionConfig retransmissionConfig) { + this.retransmissionConfig = retransmissionConfig; + return this; + } + + Builder pendingOperation(PendingOperation pendingOperation) { + this.pendingOperation = pendingOperation; + return this; + } + + MqttPendingUnsubscription build() { + return new MqttPendingUnsubscription(future, topic, unsubscribeMessage, ownerId, retransmissionConfig, pendingOperation); + } + } + } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPingHandler.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPingHandler.java index 70a4992d72..3fc2c6246e 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPingHandler.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPingHandler.java @@ -42,12 +42,11 @@ final class MqttPingHandler extends ChannelInboundHandlerAdapter { } @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - if (!(msg instanceof MqttMessage)) { + public void channelRead(ChannelHandlerContext ctx, Object msg) { + if (!(msg instanceof MqttMessage message)) { ctx.fireChannelRead(msg); return; } - MqttMessage message = (MqttMessage) msg; if (message.fixedHeader().messageType() == MqttMessageType.PINGREQ) { this.handlePingReq(ctx.channel()); } else if (message.fixedHeader().messageType() == MqttMessageType.PINGRESP) { @@ -61,28 +60,29 @@ final class MqttPingHandler extends ChannelInboundHandlerAdapter { public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { super.userEventTriggered(ctx, evt); - if (evt instanceof IdleStateEvent) { - IdleStateEvent event = (IdleStateEvent) evt; + if (evt instanceof IdleStateEvent event) { switch (event.state()) { case READER_IDLE: log.debug("[{}] No reads were performed for specified period for channel {}", event.state(), ctx.channel().id()); - this.sendPingReq(ctx.channel()); + this.sendPingReq(ctx.channel(), event); break; case WRITER_IDLE: log.debug("[{}] No writes were performed for specified period for channel {}", event.state(), ctx.channel().id()); - this.sendPingReq(ctx.channel()); + this.sendPingReq(ctx.channel(), event); break; } } } - private void sendPingReq(Channel channel) { + private void sendPingReq(Channel channel, IdleStateEvent idleEvent) { log.trace("[{}] Sending ping request", channel.id()); MqttFixedHeader fixedHeader = new MqttFixedHeader(MqttMessageType.PINGREQ, false, MqttQoS.AT_MOST_ONCE, false, 0); channel.writeAndFlush(new MqttMessage(fixedHeader)); if (this.pingRespTimeout == null) { + log.trace("[{}] Scheduling disconnect due to {}", channel.id(), idleEvent); this.pingRespTimeout = channel.eventLoop().schedule(() -> { + log.trace("[{}] Sending disconnect due to {}", channel.id(), idleEvent); MqttFixedHeader fixedHeader2 = new MqttFixedHeader(MqttMessageType.DISCONNECT, false, MqttQoS.AT_MOST_ONCE, false, 0); channel.writeAndFlush(new MqttMessage(fixedHeader2)).addListener(ChannelFutureListener.CLOSE); //TODO: what do when the connection is closed ? @@ -99,6 +99,7 @@ final class MqttPingHandler extends ChannelInboundHandlerAdapter { private void handlePingResp(Channel channel) { log.trace("[{}] Handling ping response", channel.id()); if (this.pingRespTimeout != null && !this.pingRespTimeout.isCancelled() && !this.pingRespTimeout.isDone()) { + log.trace("[{}] Cancelling disconnect due to idle event because ping response was received", channel.id()); this.pingRespTimeout.cancel(true); this.pingRespTimeout = null; } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/PendingOperation.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/PendingOperation.java index b859b216e6..07e472abb3 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/PendingOperation.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/PendingOperation.java @@ -17,6 +17,8 @@ package org.thingsboard.mqtt; public interface PendingOperation { - boolean isCanceled(); + boolean isCancelled(); + + void onMaxRetransmissionAttemptsReached(); } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/RetransmissionHandler.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/RetransmissionHandler.java index b0d9ba9002..1778abc593 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/RetransmissionHandler.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/RetransmissionHandler.java @@ -18,66 +18,119 @@ package org.thingsboard.mqtt; import io.netty.channel.EventLoop; import io.netty.handler.codec.mqtt.MqttFixedHeader; import io.netty.handler.codec.mqtt.MqttMessage; +import io.netty.handler.codec.mqtt.MqttMessageIdVariableHeader; import io.netty.handler.codec.mqtt.MqttMessageType; +import io.netty.handler.codec.mqtt.MqttPublishVariableHeader; import io.netty.handler.codec.mqtt.MqttQoS; import io.netty.util.concurrent.ScheduledFuture; import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; +@Slf4j @RequiredArgsConstructor final class RetransmissionHandler { - private volatile boolean stopped; + private final MqttClientConfig.RetransmissionConfig config; private final PendingOperation pendingOperation; + + private volatile boolean stopped; private ScheduledFuture timer; - private int timeout = 10; + private int attemptCount = 0; + + @Setter private BiConsumer handler; + + // the three fields below are used for logging only + private final String ownerId; + private String originalMessageId; + private long totalWaitingTimeMillis; + private T originalMessage; + void setOriginalMessage(T originalMessage) { + this.originalMessage = originalMessage; + var variableHeader = originalMessage.variableHeader(); + if (variableHeader instanceof MqttMessageIdVariableHeader messageIdVariableHeader) { + originalMessageId = String.valueOf(messageIdVariableHeader.messageId()); + } else if (variableHeader instanceof MqttPublishVariableHeader publishVariableHeader) { + originalMessageId = String.valueOf(publishVariableHeader.packetId()); + } else { + originalMessageId = "N/A"; + } + } + void start(EventLoop eventLoop) { if (eventLoop == null) { throw new NullPointerException("eventLoop"); } - if (this.handler == null) { + if (handler == null) { throw new NullPointerException("handler"); } - this.timeout = 10; - this.startTimer(eventLoop); + log.debug("{}MessageID[{}] Starting retransmission handler", ownerId, originalMessageId); + startTimer(eventLoop); } private void startTimer(EventLoop eventLoop) { - if (stopped || pendingOperation.isCanceled()) { + if (stopped || pendingOperation.isCancelled()) { return; } - this.timer = eventLoop.schedule(() -> { - if (stopped || pendingOperation.isCanceled()) { + + // Calculate the base delay using exponential backoff. + // For attemptCount == 0, delay = initial delay; for each subsequent attempt, the base delay doubles. + long baseDelay = config.initialDelayMillis() * (long) Math.pow(2, attemptCount); + // Apply jitter: random factor between (1 - jitterFactor) and (1 + jitterFactor). + double minFactor = 1.0 - config.jitterFactor(); + double maxFactor = 1.0 + config.jitterFactor(); + double randomFactor = config.jitterFactor() == 0 ? 1 : ThreadLocalRandom.current().nextDouble(minFactor, maxFactor); + long delayMillisWithJitter = (long) (baseDelay * randomFactor); + totalWaitingTimeMillis += delayMillisWithJitter; + + timer = eventLoop.schedule(() -> { + if (stopped || pendingOperation.isCancelled()) { return; } - this.timeout += 5; - boolean isDup = this.originalMessage.fixedHeader().isDup(); - if (this.originalMessage.fixedHeader().messageType() == MqttMessageType.PUBLISH && this.originalMessage.fixedHeader().qosLevel() != MqttQoS.AT_MOST_ONCE) { - isDup = true; + + attemptCount++; + if (attemptCount > config.maxAttempts()) { + log.debug( + "{}MessageID[{}] Gave up after {} retransmission attempts; waited a total of {} ms without receiving acknowledgement", + ownerId, originalMessageId, config.maxAttempts(), totalWaitingTimeMillis + ); + stop(); + pendingOperation.onMaxRetransmissionAttemptsReached(); + return; } - MqttFixedHeader fixedHeader = new MqttFixedHeader(this.originalMessage.fixedHeader().messageType(), isDup, this.originalMessage.fixedHeader().qosLevel(), this.originalMessage.fixedHeader().isRetain(), this.originalMessage.fixedHeader().remainingLength()); - handler.accept(fixedHeader, originalMessage); + + log.debug("{}MessageID[{}] Retransmission attempt #{} out of {}", ownerId, originalMessageId, attemptCount, config.maxAttempts()); + + var originalFixedHeader = originalMessage.fixedHeader(); + var newFixedHeader = new MqttFixedHeader( + originalFixedHeader.messageType(), + isDup(originalFixedHeader), + originalFixedHeader.qosLevel(), + originalFixedHeader.isRetain(), + originalFixedHeader.remainingLength() + ); + handler.accept(newFixedHeader, originalMessage); startTimer(eventLoop); - }, timeout, TimeUnit.SECONDS); + }, delayMillisWithJitter, TimeUnit.MILLISECONDS); + } + + private static boolean isDup(MqttFixedHeader originalFixedHeader) { + return originalFixedHeader.isDup() || (originalFixedHeader.messageType() == MqttMessageType.PUBLISH && originalFixedHeader.qosLevel() != MqttQoS.AT_MOST_ONCE); } void stop() { + log.debug("{}MessageID[{}] Stopping retransmission handler", ownerId, originalMessageId); stopped = true; - if (this.timer != null) { - this.timer.cancel(true); + if (timer != null) { + timer.cancel(true); } } - void setHandle(BiConsumer runnable) { - this.handler = runnable; - } - - void setOriginalMessage(T originalMessage) { - this.originalMessage = originalMessage; - } } diff --git a/netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttClientTest.java b/netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttClientTest.java new file mode 100644 index 0000000000..1481b354ee --- /dev/null +++ b/netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttClientTest.java @@ -0,0 +1,210 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt; + +import com.google.common.util.concurrent.Futures; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.PooledByteBufAllocator; +import io.netty.handler.codec.mqtt.MqttConnectReturnCode; +import io.netty.handler.codec.mqtt.MqttMessageType; +import io.netty.handler.codec.mqtt.MqttQoS; +import io.netty.util.ResourceLeakDetector; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.Promise; +import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.hivemq.HiveMQContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; +import org.thingsboard.common.util.AbstractListeningExecutor; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +@Slf4j +@Testcontainers +class MqttClientTest { + + final int randomPort = 0; + + @Container + HiveMQContainer broker = new HiveMQContainer(DockerImageName.parse("hivemq/hivemq-ce").withTag("2025.2")); + + MqttTestProxy proxy; + + MqttClient client; + + AbstractListeningExecutor handlerExecutor; + + @BeforeAll + static void init() { + ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); + } + + @BeforeEach + void setup() { + handlerExecutor = new AbstractListeningExecutor() { + @Override + protected int getThreadPollSize() { + return 1; + } + }; + handlerExecutor.init(); + } + + @AfterEach + void cleanup() { + if (client != null) { + client.disconnect(); + client = null; + } + if (proxy != null) { + proxy.stop(); + proxy = null; + } + handlerExecutor.destroy(); + handlerExecutor = null; + } + + @Test + void testConnectToBroker() { + // GIVEN + var clientConfig = new MqttClientConfig(); + clientConfig.setOwnerId("Test[ConnectToBroker]"); + clientConfig.setClientId("connect"); + + client = MqttClient.create(clientConfig, null, handlerExecutor); + + // WHEN + Promise connectFuture = client.connect(broker.getHost(), broker.getMqttPort()); + + // THEN + assertThat(connectFuture).isNotNull(); + + Awaitility.await("waiting for client to connect") + .atMost(Duration.ofSeconds(10L)) + .until(connectFuture::isDone); + + assertThat(connectFuture.isSuccess()).isTrue(); + + MqttConnectResult actualConnectResult = connectFuture.getNow(); + assertThat(actualConnectResult).isNotNull(); + assertThat(actualConnectResult.isSuccess()).isTrue(); + assertThat(actualConnectResult.getReturnCode()).isEqualTo(MqttConnectReturnCode.CONNECTION_ACCEPTED); + + assertThat(client.isConnected()).isTrue(); + } + + @Test + void testDisconnectDueToKeepAliveIfNoActivity() { + // GIVEN + proxy = MqttTestProxy.builder() + .localPort(randomPort) + .brokerHost(broker.getHost()) + .brokerPort(broker.getMqttPort()) + .brokerToClientInterceptor(msg -> msg.fixedHeader().messageType() != MqttMessageType.PINGRESP) // drop all ping responses to simulate broker down + .build(); + + int idleTimeoutSeconds = 2; + + var clientConfig = new MqttClientConfig(); + clientConfig.setOwnerId("Test[KeepAliveDisconnect]"); + clientConfig.setClientId("no-activity-disconnect"); + clientConfig.setTimeoutSeconds(idleTimeoutSeconds); + clientConfig.setReconnect(false); // disable auto reconnect + client = MqttClient.create(clientConfig, null, handlerExecutor); + + // WHEN-THEN + connect(broker.getHost(), proxy.getPort()); + + // no activity... + + Awaitility.await("waiting for client to disconnect") + .pollDelay(Duration.ofSeconds(idleTimeoutSeconds * 2)) // 2 seconds to wait for the first idle event and then 2 seconds for scheduled disconnect to fire + .atMost(Duration.ofSeconds(10)) + .untilAsserted(() -> assertThat(client.isConnected()).isFalse()); + } + + @Test + void testRetransmission() { + // GIVEN + proxy = MqttTestProxy.builder() + .localPort(randomPort) + .brokerHost(broker.getHost()) + .brokerPort(broker.getMqttPort()) + .brokerToClientInterceptor(msg -> msg.fixedHeader().messageType() != MqttMessageType.PUBACK) // drop all pubacks to allow retransmission to happen + .build(); + + // create client + var clientConfig = new MqttClientConfig(); + clientConfig.setOwnerId("Test[Retransmission]"); + clientConfig.setClientId("retransmission"); + clientConfig.setRetransmissionConfig(new MqttClientConfig.RetransmissionConfig(1, 1000L, 0d)); + client = MqttClient.create(clientConfig, null, handlerExecutor); + + // connect to a broker + connect(broker.getHost(), proxy.getPort()); + + // subscribe to a topic + String topic = "test-topic"; + List receivedMessages = Collections.synchronizedList(new ArrayList<>(2)); + Future subscribeFuture = client.on(topic, (__, payload) -> { + receivedMessages.add(payload); + return Futures.immediateVoidFuture(); + }); + Awaitility.await("waiting for client to subscribe to a topic") + .atMost(Duration.ofSeconds(10L)) + .until(subscribeFuture::isDone); + + // WHEN + // publish a message + ByteBuf message = PooledByteBufAllocator.DEFAULT.buffer().writeBytes("test message".getBytes(StandardCharsets.UTF_8)); + client.publish(topic, message, MqttQoS.AT_LEAST_ONCE); + + // THEN + // wait enough time so that retransmission happens and stops + // if retransmission works incorrectly waiting 10 seconds allows for additional retransmissions to happen + try { + Awaitility.await("wait up to 10s, stop early if too many messages") + .atMost(Duration.ofSeconds(10L)) + .pollInterval(Duration.ofMillis(100)) + .until(() -> receivedMessages.size() > 2); + } catch (ConditionTimeoutException __) { + // didn't exceed 2 messages + } + + assertThat(receivedMessages).size().describedAs("incorrect number of messages received, expected 2 (original plus one retransmitted)").isEqualTo(2); + } + + private void connect(String host, int port) { + Promise connectFuture = client.connect(host, port); + Awaitility.await("waiting for client to connect") + .atMost(Duration.ofSeconds(10L)) + .until(connectFuture::isSuccess); + } + +} diff --git a/netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttPingHandlerTest.java b/netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttPingHandlerTest.java deleted file mode 100644 index 83e3b1c8d5..0000000000 --- a/netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttPingHandlerTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.mqtt; - -import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.DefaultEventLoop; -import io.netty.handler.timeout.IdleStateEvent; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.concurrent.TimeUnit; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.after; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -class MqttPingHandlerTest { - - static final int KEEP_ALIVE_SECONDS = 0; - static final int PROCESS_SEND_DISCONNECT_MSG_TIME_MS = 500; - - MqttPingHandler mqttPingHandler; - - @BeforeEach - void setUp() { - mqttPingHandler = new MqttPingHandler(KEEP_ALIVE_SECONDS); - } - - @Test - void givenChannelReaderIdleState_whenNoPingResponse_thenDisconnectClient() throws Exception { - ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); - Channel channel = mock(Channel.class); - when(ctx.channel()).thenReturn(channel); - when(channel.eventLoop()).thenReturn(new DefaultEventLoop()); - ChannelFuture channelFuture = mock(ChannelFuture.class); - when(channel.writeAndFlush(any())).thenReturn(channelFuture); - - mqttPingHandler.userEventTriggered(ctx, IdleStateEvent.FIRST_READER_IDLE_STATE_EVENT); - verify( - channelFuture, - after(TimeUnit.SECONDS.toMillis(KEEP_ALIVE_SECONDS) + PROCESS_SEND_DISCONNECT_MSG_TIME_MS) - ).addListener(eq(ChannelFutureListener.CLOSE)); - } -} \ No newline at end of file diff --git a/netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttTestProxy.java b/netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttTestProxy.java new file mode 100644 index 0000000000..4a10fc3bfb --- /dev/null +++ b/netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttTestProxy.java @@ -0,0 +1,202 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt; + +import io.netty.bootstrap.Bootstrap; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.codec.mqtt.MqttDecoder; +import io.netty.handler.codec.mqtt.MqttEncoder; +import io.netty.handler.codec.mqtt.MqttMessage; +import io.netty.util.ReferenceCountUtil; +import lombok.extern.slf4j.Slf4j; + +import java.net.InetSocketAddress; +import java.util.function.Predicate; + +@Slf4j +public class MqttTestProxy { + + private final EventLoopGroup bossGroup; + private final EventLoopGroup workerGroup; + + private Channel clientToProxyChannel; + private Channel proxyToBrokerChannel; + + private final int assignedPort; + + private boolean stopped; + + private final Predicate brokerToClientInterceptor; + + private MqttTestProxy(Builder builder) { + log.info("Starting MQTT proxy..."); + + brokerToClientInterceptor = builder.brokerToClientInterceptor != null ? builder.brokerToClientInterceptor : msg -> true; + bossGroup = new NioEventLoopGroup(1); + workerGroup = new NioEventLoopGroup(1); + + ServerBootstrap proxyBootstrap = new ServerBootstrap(); + proxyBootstrap.group(bossGroup, workerGroup) + .channel(NioServerSocketChannel.class) + .childHandler(new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel channel) { + clientToProxyChannel = channel; + clientToProxyChannel.config().setAutoRead(false); // do not accept data before we connected to a broker + + connectToBroker(builder.brokerHost, builder.brokerPort).addListener(future -> { + if (future.isSuccess()) { + clientToProxyChannel.pipeline().addLast("mqttDecoder", new MqttDecoder()); + clientToProxyChannel.pipeline().addLast("mqttToBroker", new MqttRelayHandler(proxyToBrokerChannel, null)); + clientToProxyChannel.pipeline().addLast("mqttEncoder", MqttEncoder.INSTANCE); + + clientToProxyChannel.config().setAutoRead(true); // start accepting data for a client + } else { + log.error("Failed to connect to broker", future.cause()); + clientToProxyChannel.close(); + } + }); + } + }); + + try { + Channel proxyChannel = proxyBootstrap.bind(builder.localPort).sync().channel(); + assignedPort = ((InetSocketAddress) proxyChannel.localAddress()).getPort(); + } catch (Exception e) { + log.error("Failed to start MQTT proxy", e); + throw new RuntimeException("Failed to start MQTT proxy", e); + } + + log.info("MQTT proxy started on port {}", assignedPort); + } + + private ChannelFuture connectToBroker(String brokerHost, int brokerPort) { + Bootstrap proxyToBrokerBootstrap = new Bootstrap(); + proxyToBrokerBootstrap.group(workerGroup) + .channel(NioSocketChannel.class) + .handler(new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel channel) { + proxyToBrokerChannel = channel; + proxyToBrokerChannel.pipeline().addLast(new MqttDecoder()); + proxyToBrokerChannel.pipeline().addLast("mqttToClient", new MqttRelayHandler(clientToProxyChannel, brokerToClientInterceptor)); + proxyToBrokerChannel.pipeline().addLast(MqttEncoder.INSTANCE); + } + }); + return proxyToBrokerBootstrap.connect(brokerHost, brokerPort); + } + + private static class MqttRelayHandler extends SimpleChannelInboundHandler { + + private final Channel targetChannel; + private final Predicate interceptor; + + private MqttRelayHandler(Channel targetChannel, Predicate interceptor) { + this.targetChannel = targetChannel; + this.interceptor = interceptor; + } + + @Override + protected void channelRead0(ChannelHandlerContext ctx, MqttMessage msg) { + log.debug("Received message: {}", msg.fixedHeader().messageType()); + if (interceptor == null || interceptor.test(msg)) { + if (targetChannel.isActive()) { + targetChannel.writeAndFlush(ReferenceCountUtil.retain(msg)); + } + } else { + log.info("Dropping message: {}", msg.fixedHeader().messageType()); + } + } + + } + + public void stop() { + if (stopped) { + log.info("MQTT proxy was already stopped"); + return; + } + + stopped = true; + + log.info("Stopping MQTT proxy..."); + + if (clientToProxyChannel != null) { + clientToProxyChannel.close(); + } + if (proxyToBrokerChannel != null) { + proxyToBrokerChannel.close(); + } + if (bossGroup != null) { + bossGroup.shutdownGracefully(); + } + if (workerGroup != null) { + workerGroup.shutdownGracefully(); + } + + log.info("MQTT proxy stopped"); + } + + public int getPort() { + return assignedPort; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + + private int localPort; + private String brokerHost; + private int brokerPort; + private Predicate brokerToClientInterceptor; + + public Builder localPort(int localPort) { + this.localPort = localPort; + return this; + } + + public Builder brokerHost(String brokerHost) { + this.brokerHost = brokerHost; + return this; + } + + public Builder brokerPort(int brokerPort) { + this.brokerPort = brokerPort; + return this; + } + + public Builder brokerToClientInterceptor(Predicate interceptor) { + this.brokerToClientInterceptor = interceptor; + return this; + } + + public MqttTestProxy build() { + return new MqttTestProxy(this); + } + + } +} diff --git a/netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/MqttIntegrationTest.java b/netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/MqttIntegrationTest.java deleted file mode 100644 index db177c84b6..0000000000 --- a/netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/MqttIntegrationTest.java +++ /dev/null @@ -1,151 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.mqtt.integration; - -import io.netty.buffer.Unpooled; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.handler.codec.mqtt.MqttMessageType; -import io.netty.handler.codec.mqtt.MqttQoS; -import io.netty.util.concurrent.Future; -import io.netty.util.concurrent.Promise; -import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.parallel.ResourceLock; -import org.thingsboard.common.util.AbstractListeningExecutor; -import org.thingsboard.mqtt.MqttClient; -import org.thingsboard.mqtt.MqttClientConfig; -import org.thingsboard.mqtt.MqttConnectResult; -import org.thingsboard.mqtt.integration.server.MqttServer; - -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -@ResourceLock("port8885") // test MQTT server port -@Slf4j -public class MqttIntegrationTest { - - static final String MQTT_HOST = "localhost"; - static final int KEEPALIVE_TIMEOUT_SECONDS = 2; - static final long RECONNECT_DELAY_SECONDS = 10L; - - EventLoopGroup eventLoopGroup; - MqttServer mqttServer; - - MqttClient mqttClient; - - AbstractListeningExecutor handlerExecutor; - - @BeforeEach - public void init() throws Exception { - this.handlerExecutor = new AbstractListeningExecutor() { - @Override - protected int getThreadPollSize() { - return 4; - } - }; - handlerExecutor.init(); - - this.eventLoopGroup = new NioEventLoopGroup(); - - this.mqttServer = new MqttServer(); - this.mqttServer.init(); - } - - @AfterEach - public void destroy() throws InterruptedException { - if (this.mqttClient != null) { - this.mqttClient.disconnect(); - } - if (this.mqttServer != null) { - this.mqttServer.shutdown(); - } - if (this.eventLoopGroup != null) { - this.eventLoopGroup.shutdownGracefully(0, 0, TimeUnit.MILLISECONDS); - } - if (this.handlerExecutor != null) { - this.handlerExecutor.destroy(); - } - } - - @Test - public void givenActiveMqttClient_whenNoActivityForKeepAliveTimeout_thenDisconnectClient() throws Throwable { - //given - this.mqttClient = initClient(); - - log.warn("Sending publish messages..."); - CountDownLatch latch = new CountDownLatch(3); - for (int i = 0; i < 3; i++) { - Thread.sleep(30); - Future pubFuture = publishMsg(); - pubFuture.addListener(future -> latch.countDown()); - } - - log.warn("Waiting for messages acknowledgments..."); - boolean awaitResult = latch.await(10, TimeUnit.SECONDS); - Assertions.assertTrue(awaitResult); - log.warn("Messages are delivered successfully..."); - - //when - log.warn("Starting idle period..."); - Thread.sleep(5000); - - //then - List allReceivedEvents = this.mqttServer.getEventsFromClient(); - long disconnectCount = allReceivedEvents.stream().filter(type -> type == MqttMessageType.DISCONNECT).count(); - - Assertions.assertEquals(1, disconnectCount); - } - - private Future publishMsg() { - return this.mqttClient.publish( - "test/topic", - Unpooled.wrappedBuffer("payload".getBytes(StandardCharsets.UTF_8)), - MqttQoS.AT_MOST_ONCE); - } - - private MqttClient initClient() throws Exception { - MqttClientConfig config = new MqttClientConfig(); - config.setOwnerId("MqttIntegrationTest"); - config.setTimeoutSeconds(KEEPALIVE_TIMEOUT_SECONDS); - config.setReconnectDelay(RECONNECT_DELAY_SECONDS); - MqttClient client = MqttClient.create(config, null, handlerExecutor); - client.setEventLoop(this.eventLoopGroup); - Promise connectFuture = client.connect(MQTT_HOST, this.mqttServer.getMqttPort()); - - String hostPort = MQTT_HOST + ":" + this.mqttServer.getMqttPort(); - MqttConnectResult result; - try { - result = connectFuture.get(10, TimeUnit.SECONDS); - } catch (TimeoutException ex) { - connectFuture.cancel(true); - client.disconnect(); - throw new RuntimeException(String.format("Failed to connect to MQTT server at %s.", hostPort)); - } - if (!result.isSuccess()) { - connectFuture.cancel(true); - client.disconnect(); - throw new RuntimeException(String.format("Failed to connect to MQTT server at %s. Result code is: %s", hostPort, result.getReturnCode())); - } - return client; - } -} \ No newline at end of file diff --git a/netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttServer.java b/netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttServer.java deleted file mode 100644 index ca4fb677dc..0000000000 --- a/netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttServer.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.mqtt.integration.server; - -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.Channel; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.ChannelPipeline; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.handler.codec.mqtt.MqttDecoder; -import io.netty.handler.codec.mqtt.MqttEncoder; -import io.netty.handler.codec.mqtt.MqttMessageType; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; - -@Slf4j -public class MqttServer { - - @Getter - private final List eventsFromClient = new CopyOnWriteArrayList<>(); - @Getter - private final int mqttPort = 8885; - - private Channel serverChannel; - private EventLoopGroup bossGroup; - private EventLoopGroup workerGroup; - - public void init() throws Exception { - log.info("Starting MQTT server on port {}...", mqttPort); - bossGroup = new NioEventLoopGroup(); - workerGroup = new NioEventLoopGroup(); - ServerBootstrap b = new ServerBootstrap(); - b.group(bossGroup, workerGroup) - .channel(NioServerSocketChannel.class) - .childHandler(new ChannelInitializer() { - @Override - protected void initChannel(SocketChannel ch) throws Exception { - ChannelPipeline pipeline = ch.pipeline(); - pipeline.addLast("decoder", new MqttDecoder(65536)); - pipeline.addLast("encoder", MqttEncoder.INSTANCE); - - MqttTransportHandler handler = new MqttTransportHandler(eventsFromClient); - - pipeline.addLast(handler); - ch.closeFuture().addListener(handler); - } - }) - .childOption(ChannelOption.SO_KEEPALIVE, true); - - serverChannel = b.bind(mqttPort).sync().channel(); - log.info("Mqtt transport started!"); - } - - public void shutdown() throws InterruptedException { - log.info("Stopping MQTT transport!"); - try { - serverChannel.close().sync(); - } finally { - workerGroup.shutdownGracefully(); - bossGroup.shutdownGracefully(); - } - log.info("MQTT transport stopped!"); - } -} diff --git a/netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttTransportHandler.java b/netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttTransportHandler.java deleted file mode 100644 index 5c433d7069..0000000000 --- a/netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttTransportHandler.java +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.mqtt.integration.server; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.handler.codec.mqtt.MqttConnAckMessage; -import io.netty.handler.codec.mqtt.MqttConnAckVariableHeader; -import io.netty.handler.codec.mqtt.MqttConnectMessage; -import io.netty.handler.codec.mqtt.MqttConnectReturnCode; -import io.netty.handler.codec.mqtt.MqttFixedHeader; -import io.netty.handler.codec.mqtt.MqttMessage; -import io.netty.handler.codec.mqtt.MqttMessageIdVariableHeader; -import io.netty.handler.codec.mqtt.MqttMessageType; -import io.netty.handler.codec.mqtt.MqttPubAckMessage; -import io.netty.handler.codec.mqtt.MqttPublishMessage; -import io.netty.util.ReferenceCountUtil; -import io.netty.util.concurrent.Future; -import io.netty.util.concurrent.GenericFutureListener; -import lombok.extern.slf4j.Slf4j; - -import java.util.List; -import java.util.UUID; - -import static io.netty.handler.codec.mqtt.MqttMessageType.CONNACK; -import static io.netty.handler.codec.mqtt.MqttMessageType.CONNECT; -import static io.netty.handler.codec.mqtt.MqttMessageType.DISCONNECT; -import static io.netty.handler.codec.mqtt.MqttMessageType.PINGREQ; -import static io.netty.handler.codec.mqtt.MqttMessageType.PUBACK; -import static io.netty.handler.codec.mqtt.MqttMessageType.PUBLISH; -import static io.netty.handler.codec.mqtt.MqttQoS.AT_MOST_ONCE; - -@Slf4j -public class MqttTransportHandler extends ChannelInboundHandlerAdapter implements GenericFutureListener> { - - private final List eventsFromClient; - private final UUID sessionId; - - MqttTransportHandler(List eventsFromClient) { - this.sessionId = UUID.randomUUID(); - this.eventsFromClient = eventsFromClient; - } - - @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) { - log.trace("[{}] Processing msg: {}", sessionId, msg); - try { - if (msg instanceof MqttMessage) { - MqttMessage message = (MqttMessage) msg; - if (message.decoderResult().isSuccess()) { - processMqttMsg(ctx, message); - } else { - log.error("[{}] Message decoding failed: {}", sessionId, message.decoderResult().cause().getMessage()); - ctx.close(); - } - } else { - log.debug("[{}] Received non mqtt message: {}", sessionId, msg.getClass().getSimpleName()); - ctx.close(); - } - } finally { - ReferenceCountUtil.safeRelease(msg); - } - } - - void processMqttMsg(ChannelHandlerContext ctx, MqttMessage msg) { - if (msg.fixedHeader() == null) { - ctx.close(); - return; - } - switch (msg.fixedHeader().messageType()) { - case CONNECT: - eventsFromClient.add(CONNECT); - processConnect(ctx, (MqttConnectMessage) msg); - break; - case DISCONNECT: - eventsFromClient.add(DISCONNECT); - ctx.close(); - break; - case PUBLISH: - // QoS 0 and 1 supported only here - eventsFromClient.add(PUBLISH); - MqttPublishMessage mqttPubMsg = (MqttPublishMessage) msg; - ack(ctx, mqttPubMsg.variableHeader().packetId()); - break; - case PINGREQ: - // We will not handle PINGREQ and will not send any PINGRESP to simulate the MQTT server is down - eventsFromClient.add(PINGREQ); - break; - default: - break; - } - } - - void processConnect(ChannelHandlerContext ctx, MqttConnectMessage msg) { - String userName = msg.payload().userName(); - String clientId = msg.payload().clientIdentifier(); - - log.warn("[{}][{}] Processing connect msg for client: {}!", sessionId, userName, clientId); - ctx.writeAndFlush(createMqttConnAckMsg(msg)); - } - - private MqttConnAckMessage createMqttConnAckMsg(MqttConnectMessage msg) { - MqttFixedHeader mqttFixedHeader = - new MqttFixedHeader(CONNACK, false, AT_MOST_ONCE, false, 0); - MqttConnAckVariableHeader mqttConnAckVariableHeader = - new MqttConnAckVariableHeader(MqttConnectReturnCode.CONNECTION_ACCEPTED, !msg.variableHeader().isCleanSession()); - return new MqttConnAckMessage(mqttFixedHeader, mqttConnAckVariableHeader); - } - - private void ack(ChannelHandlerContext ctx, int msgId) { - if (msgId > 0) { - ctx.writeAndFlush(createMqttPubAckMsg(msgId)); - } - } - - public static MqttPubAckMessage createMqttPubAckMsg(int requestId) { - MqttFixedHeader mqttFixedHeader = - new MqttFixedHeader(PUBACK, false, AT_MOST_ONCE, false, 0); - MqttMessageIdVariableHeader mqttMsgIdVariableHeader = - MqttMessageIdVariableHeader.from(requestId); - return new MqttPubAckMessage(mqttFixedHeader, mqttMsgIdVariableHeader); - } - - @Override - public void operationComplete(Future future) { - log.trace("[{}] Channel closed!", sessionId); - } -} diff --git a/netty-mqtt/src/test/resources/junit-platform.properties b/netty-mqtt/src/test/resources/junit-platform.properties deleted file mode 100644 index f2ed301920..0000000000 --- a/netty-mqtt/src/test/resources/junit-platform.properties +++ /dev/null @@ -1,3 +0,0 @@ -junit.jupiter.execution.parallel.enabled = true -junit.jupiter.execution.parallel.mode.default = concurrent -junit.jupiter.execution.parallel.mode.classes.default = concurrent diff --git a/pom.xml b/pom.xml index 72195b1e4f..70e0777462 100755 --- a/pom.xml +++ b/pom.xml @@ -1957,6 +1957,12 @@ ${testcontainers.version} test + + org.testcontainers + hivemq + ${testcontainers.version} + test + org.springframework.data spring-data-redis diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/MqttClientSettings.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/MqttClientSettings.java new file mode 100644 index 0000000000..4ac05b57d0 --- /dev/null +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/MqttClientSettings.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.api; + +public interface MqttClientSettings { + + int getRetransmissionMaxAttempts(); + + long getRetransmissionInitialDelayMillis(); + + double getRetransmissionJitterFactor(); + +} diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index b66c9e13d5..7989b8f9ce 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -416,4 +416,9 @@ public interface TbContext { EventService getEventService(); AuditLogService getAuditLogService(); + + // Configuration parameters for the MQTT client that is used in the MQTT node and Azure IoT hub node + + MqttClientSettings getMqttClientSettings(); + } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java index 4d99951e1a..28a9e1ff4b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java @@ -26,6 +26,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.mqtt.MqttClient; import org.thingsboard.mqtt.MqttClientConfig; import org.thingsboard.mqtt.MqttConnectResult; +import org.thingsboard.rule.engine.api.MqttClientSettings; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeConfiguration; @@ -126,6 +127,13 @@ public class TbMqttNode extends TbAbstractExternalNode { } config.setCleanSession(this.mqttNodeConfiguration.isCleanSession()); + MqttClientSettings mqttClientSettings = ctx.getMqttClientSettings(); + config.setRetransmissionConfig(new MqttClientConfig.RetransmissionConfig( + mqttClientSettings.getRetransmissionMaxAttempts(), + mqttClientSettings.getRetransmissionInitialDelayMillis(), + mqttClientSettings.getRetransmissionJitterFactor() + )); + prepareMqttClientConfig(config); MqttClient client = getMqttClient(ctx, config); client.setEventLoop(ctx.getSharedEventLoop()); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mqtt/TbMqttNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mqtt/TbMqttNodeTest.java index f6ccfbca6f..bf650af8bb 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mqtt/TbMqttNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mqtt/TbMqttNodeTest.java @@ -40,6 +40,7 @@ import org.thingsboard.mqtt.MqttClient; import org.thingsboard.mqtt.MqttClientConfig; import org.thingsboard.mqtt.MqttConnectResult; import org.thingsboard.rule.engine.AbstractRuleNodeUpgradeTest; +import org.thingsboard.rule.engine.api.MqttClientSettings; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; @@ -80,6 +81,7 @@ import static org.mockito.BDDMockito.spy; import static org.mockito.BDDMockito.then; import static org.mockito.BDDMockito.willAnswer; import static org.mockito.BDDMockito.willReturn; +import static org.mockito.Mockito.lenient; @ExtendWith(MockitoExtension.class) public class TbMqttNodeTest extends AbstractRuleNodeUpgradeTest { @@ -106,6 +108,22 @@ public class TbMqttNodeTest extends AbstractRuleNodeUpgradeTest { protected void setUp() { mqttNode = spy(new TbMqttNode()); mqttNodeConfig = new TbMqttNodeConfiguration().defaultConfiguration(); + lenient().when(ctxMock.getMqttClientSettings()).thenReturn(new MqttClientSettings() { + @Override + public int getRetransmissionMaxAttempts() { + return 3; + } + + @Override + public long getRetransmissionInitialDelayMillis() { + return 5000L; + } + + @Override + public double getRetransmissionJitterFactor() { + return 0.15; + } + }); } @Test From d6aa53a858e90b66b2f5c9a050308f8a67a0cdc3 Mon Sep 17 00:00:00 2001 From: yuliaklochai Date: Fri, 2 May 2025 12:42:14 +0300 Subject: [PATCH 31/40] UI: added trendz settings tab --- ui-ngx/src/app/core/auth/auth.models.ts | 2 + ui-ngx/src/app/core/auth/auth.reducer.ts | 4 +- .../app/core/http/trendz-settings.service.ts | 39 ++++++++ ui-ngx/src/app/core/services/menu.models.ts | 19 +++- .../home/pages/admin/admin-routing.module.ts | 13 +++ .../modules/home/pages/admin/admin.module.ts | 4 +- .../admin/trendz-settings.component.html | 51 ++++++++++ .../admin/trendz-settings.component.scss | 36 +++++++ .../pages/admin/trendz-settings.component.ts | 95 +++++++++++++++++++ ui-ngx/src/app/shared/models/constants.ts | 1 + ui-ngx/src/app/shared/models/icon.models.ts | 14 ++- .../shared/models/trendz-settings.models.ts | 25 +++++ .../assets/locale/locale.constant-en_US.json | 6 +- 13 files changed, 302 insertions(+), 7 deletions(-) create mode 100644 ui-ngx/src/app/core/http/trendz-settings.service.ts create mode 100644 ui-ngx/src/app/modules/home/pages/admin/trendz-settings.component.html create mode 100644 ui-ngx/src/app/modules/home/pages/admin/trendz-settings.component.scss create mode 100644 ui-ngx/src/app/modules/home/pages/admin/trendz-settings.component.ts create mode 100644 ui-ngx/src/app/shared/models/trendz-settings.models.ts diff --git a/ui-ngx/src/app/core/auth/auth.models.ts b/ui-ngx/src/app/core/auth/auth.models.ts index e5cc1424ab..142d845cf4 100644 --- a/ui-ngx/src/app/core/auth/auth.models.ts +++ b/ui-ngx/src/app/core/auth/auth.models.ts @@ -16,6 +16,7 @@ import { AuthUser, User } from '@shared/models/user.model'; import { UserSettings } from '@shared/models/user-settings.models'; +import { TrendzSettings } from '@shared/models/trendz-settings.models'; export interface SysParamsState { userTokenAccessEnabled: boolean; @@ -32,6 +33,7 @@ export interface SysParamsState { maxArgumentsPerCF: number; ruleChainDebugPerTenantLimitsConfiguration?: string; calculatedFieldDebugPerTenantLimitsConfiguration?: string; + trendzSettings: TrendzSettings; } export interface SysParams extends SysParamsState { diff --git a/ui-ngx/src/app/core/auth/auth.reducer.ts b/ui-ngx/src/app/core/auth/auth.reducer.ts index 3ecf70074c..fde778284d 100644 --- a/ui-ngx/src/app/core/auth/auth.reducer.ts +++ b/ui-ngx/src/app/core/auth/auth.reducer.ts @@ -17,6 +17,7 @@ import { AuthPayload, AuthState } from './auth.models'; import { AuthActions, AuthActionTypes } from './auth.actions'; import { initialUserSettings, UserSettings } from '@shared/models/user-settings.models'; +import { initialTrendzSettings } from '@shared/models/trendz-settings.models'; import { unset } from '@core/utils'; const emptyUserAuthState: AuthPayload = { @@ -34,7 +35,8 @@ const emptyUserAuthState: AuthPayload = { maxArgumentsPerCF: 0, maxDataPointsPerRollingArg: 0, maxDebugModeDurationMinutes: 0, - userSettings: initialUserSettings + userSettings: initialUserSettings, + trendzSettings: initialTrendzSettings }; export const initialState: AuthState = { diff --git a/ui-ngx/src/app/core/http/trendz-settings.service.ts b/ui-ngx/src/app/core/http/trendz-settings.service.ts new file mode 100644 index 0000000000..4d965f920e --- /dev/null +++ b/ui-ngx/src/app/core/http/trendz-settings.service.ts @@ -0,0 +1,39 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Injectable } from '@angular/core'; +import { Observable } from 'rxjs'; +import { HttpClient } from '@angular/common/http'; +import { TrendzSettings } from '@shared/models/trendz-settings.models'; +import { defaultHttpOptionsFromConfig } from '@core/http/http-utils'; + +@Injectable({ + providedIn: 'root' +}) +export class TrendzSettingsService { + + constructor( + private http: HttpClient + ) {} + + public getTrendzSettings(): Observable { + return this.http.get(`/api/trendz/settings`, defaultHttpOptionsFromConfig({ignoreLoading: true, ignoreErrors: true})) + } + + public saveTrendzSettings(trendzSettings: TrendzSettings): Observable { + return this.http.post(`/api/trendz/settings`, trendzSettings, defaultHttpOptionsFromConfig({ignoreLoading: true, ignoreErrors: true})) + } +} diff --git a/ui-ngx/src/app/core/services/menu.models.ts b/ui-ngx/src/app/core/services/menu.models.ts index 4775a3a771..607c5c6dff 100644 --- a/ui-ngx/src/app/core/services/menu.models.ts +++ b/ui-ngx/src/app/core/services/menu.models.ts @@ -104,7 +104,8 @@ export enum MenuId { features = 'features', otaUpdates = 'otaUpdates', version_control = 'version_control', - api_usage = 'api_usage' + api_usage = 'api_usage', + trendz_settings = 'trendz_settings' } declare type MenuFilter = (authState: AuthState) => boolean; @@ -684,6 +685,17 @@ export const menuSectionMap = new Map([ path: '/usage', icon: 'insert_chart' } + ], + [ + MenuId.trendz_settings, + { + id: MenuId.trendz_settings, + name: 'admin.trendz', + fullName: 'admin.trendz-settings', + type: 'link', + path: '/settings/trendz', + icon: 'trendz-settings' + } ] ]); @@ -843,7 +855,8 @@ const defaultUserMenuMap = new Map([ {id: MenuId.home_settings}, {id: MenuId.notification_settings}, {id: MenuId.repository_settings}, - {id: MenuId.auto_commit_settings} + {id: MenuId.auto_commit_settings}, + {id: MenuId.trendz_settings} ] }, { @@ -946,7 +959,7 @@ const defaultHomeSectionMap = new Map([ }, { name: 'admin.system-settings', - places: [MenuId.home_settings, MenuId.resources_library, MenuId.repository_settings, MenuId.auto_commit_settings] + places: [MenuId.home_settings, MenuId.resources_library, MenuId.repository_settings, MenuId.auto_commit_settings, MenuId.trendz_settings] } ] ], diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts index 8bd724de4a..2836224a9a 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts @@ -46,6 +46,7 @@ import { ScadaSymbolData } from '@home/pages/scada-symbol/scada-symbol-editor.mo import { MenuId } from '@core/services/menu.models'; import { catchError } from 'rxjs/operators'; import { JsLibraryTableConfigResolver } from '@home/pages/admin/resource/js-library-table-config.resolver'; +import { TrendzSettingsComponent } from '@home/pages/admin/trendz-settings.component'; export const scadaSymbolResolver: ResolveFn = (route: ActivatedRouteSnapshot, @@ -349,6 +350,18 @@ const routes: Routes = [ } } }, + { + path: 'trendz', + component: TrendzSettingsComponent, + canDeactivate: [ConfirmOnExitGuard], + data: { + auth: [Authority.TENANT_ADMIN], + title: 'admin.trendz-settings', + breadcrumb: { + menuId: MenuId.trendz_settings + } + } + }, { path: 'security-settings', redirectTo: '/security-settings/general' diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts index 63878ebac9..a5f18122fd 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts @@ -37,6 +37,7 @@ import { OAuth2Module } from '@home/pages/admin/oauth2/oauth2.module'; import { JsLibraryTableHeaderComponent } from '@home/pages/admin/resource/js-library-table-header.component'; import { JsResourceComponent } from '@home/pages/admin/resource/js-resource.component'; import { NgxFlowModule } from '@flowjs/ngx-flow'; +import { TrendzSettingsComponent } from '@home/pages/admin/trendz-settings.component'; @NgModule({ declarations: @@ -55,7 +56,8 @@ import { NgxFlowModule } from '@flowjs/ngx-flow'; QueueComponent, RepositoryAdminSettingsComponent, AutoCommitAdminSettingsComponent, - TwoFactorAuthSettingsComponent + TwoFactorAuthSettingsComponent, + TrendzSettingsComponent ], imports: [ CommonModule, diff --git a/ui-ngx/src/app/modules/home/pages/admin/trendz-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/trendz-settings.component.html new file mode 100644 index 0000000000..45ba62dace --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/trendz-settings.component.html @@ -0,0 +1,51 @@ + +
+ + + + admin.trendz-settings + + +
+ + + +
+ +
+
+
+ + admin.trendz-url + + + + {{ 'admin.trendz-enable' | translate }} + +
+
+ +
+
+
+ + +
diff --git a/ui-ngx/src/app/modules/home/pages/admin/trendz-settings.component.scss b/ui-ngx/src/app/modules/home/pages/admin/trendz-settings.component.scss new file mode 100644 index 0000000000..cbb8e698bd --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/trendz-settings.component.scss @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + @import "../../../../../scss/constants"; + +:host { + .mat-mdc-card-header { + min-height: 64px; + } + + .tb-trendz-section { + margin: 16px 0; + } + + .tb-trendz-url { + @media #{$mat-gt-sm} { + padding-right: 12px; + } + + @media #{$mat-lt-md} { + padding-bottom: 12px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/pages/admin/trendz-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/trendz-settings.component.ts new file mode 100644 index 0000000000..ffd2898d2a --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/trendz-settings.component.ts @@ -0,0 +1,95 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { select, Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { TrendzSettingsService } from '@core/http/trendz-settings.service'; +import { TrendzSettings } from '@shared/models/trendz-settings.models'; +import { isDefinedAndNotNull } from '@core/utils'; + +@Component({ + selector: 'tb-trendz-settings', + templateUrl: './trendz-settings.component.html', + styleUrls: ['./trendz-settings.component.scss', './settings-card.scss'] +}) +export class TrendzSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { + + trendzSettingsForm: FormGroup; + + constructor(protected store: Store, + private fb: FormBuilder, + private trendzSettingsService: TrendzSettingsService) { + super(store); + } + + ngOnInit() { + this.trendzSettingsForm = this.fb.group({ + trendzUrl: [null, [Validators.pattern(/^(https?:\/\/)[^\s/$.?#].[^\s]*$/i)]], + isTrendzEnabled: [false] + }); + + this.trendzSettingsService.getTrendzSettings().subscribe((trendzSettings) => { + this.setTrendzSettings(trendzSettings); + }); + + this.trendzSettingsForm.get('isTrendzEnabled').valueChanges + .subscribe((enabled: boolean) => this.toggleUrlRequired(enabled)); + } + + toggleUrlRequired(enabled: boolean) { + const trendzUrlControl = this.trendzSettingsForm.get('trendzUrl')!; + const validators = [Validators.pattern(/^(https?:\/\/)[^\s/$.?#].[^\s]*$/i)]; + + if (enabled) { + validators.push(Validators.required); + } + + trendzUrlControl.setValidators(validators); + trendzUrlControl.updateValueAndValidity(); + } + + setTrendzSettings(trendzSettings: TrendzSettings) { + this.trendzSettingsForm.reset({ + trendzUrl: trendzSettings?.baseUrl, + isTrendzEnabled: isDefinedAndNotNull(trendzSettings?.enabled) ? + trendzSettings?.enabled : false + }); + + this.toggleUrlRequired(this.trendzSettingsForm.get('isTrendzEnabled').value); + } + + confirmForm(): FormGroup { + return this.trendzSettingsForm; + } + + save(): void { + const trendzUrl = this.trendzSettingsForm.get('trendzUrl').value; + const isTrendzEnabled = this.trendzSettingsForm.get('isTrendzEnabled').value; + + const trendzSettings: TrendzSettings = { + baseUrl: trendzUrl, + enabled: isTrendzEnabled + }; + + this.trendzSettingsService.saveTrendzSettings(trendzSettings).subscribe(() => { + this.setTrendzSettings(trendzSettings); + }) + } +} diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index e1306e577c..b2f96409dc 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -200,6 +200,7 @@ export const HelpLinks = { mobileQrCode: `${helpBaseUrl}/docs${docPlatformPrefix}/user-guide/ui/mobile-qr-code/`, calculatedField: `${helpBaseUrl}/docs${docPlatformPrefix}/`, timewindowSettings: `${helpBaseUrl}/docs${docPlatformPrefix}/user-guide/dashboards/#time-window`, + trendzSettings: `${helpBaseUrl}/docs/trendz/` } }; /* eslint-enable max-len */ diff --git a/ui-ngx/src/app/shared/models/icon.models.ts b/ui-ngx/src/app/shared/models/icon.models.ts index c9de8ec3da..6643bd9c7f 100644 --- a/ui-ngx/src/app/shared/models/icon.models.ts +++ b/ui-ngx/src/app/shared/models/icon.models.ts @@ -62,7 +62,19 @@ export const svgIcons: {[key: string]: string} = { '4.6760606 4.678212,7.3604329 7.3397982,4.6839955 4.6657413,2.0041717 6.6653477,2.2309572e-4 9.3360035,2.6766286 11.997681,' + '0 14.659287,2.6765011 Z m -5.332255,4.0079963 1.999613,2.003945 -7.99844,8.0158157 -1.9996133,-2.004017 z m 1.676684,4.3522483 ' + '1.999613,2.0039454 -6.6654242,6.679793 -1.9996133,-2.003874 z m 2.988987,7.0033574 -1.999544,-2.003945 -4.6658108,4.675848 ' + - '1.9996128,2.004015 z"/>' + '1.9996128,2.004015 z"/>', + 'trendz-settings': '' + + '' }; export const svgIconsUrl: { [key: string]: string } = { diff --git a/ui-ngx/src/app/shared/models/trendz-settings.models.ts b/ui-ngx/src/app/shared/models/trendz-settings.models.ts new file mode 100644 index 0000000000..e09797bd7e --- /dev/null +++ b/ui-ngx/src/app/shared/models/trendz-settings.models.ts @@ -0,0 +1,25 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +export interface TrendzSettings { + baseUrl: string, + enabled: boolean +} + +export const initialTrendzSettings: TrendzSettings = { + baseUrl: null, + enabled: false +} diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index b160a44b0e..8255d4e10b 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -545,7 +545,11 @@ "slack-settings": "Slack settings", "mobile-settings": "Mobile settings", "firebase-service-account-file": "Firebase service account credentials JSON file", - "select-firebase-service-account-file": "Drag and drop your Firebase service account credentials file or " + "select-firebase-service-account-file": "Drag and drop your Firebase service account credentials file or ", + "trendz": "Trendz", + "trendz-settings": "Trendz settings", + "trendz-url": "Trendz URL", + "trendz-enable": "Enable Trendz" }, "alarm": { "alarm": "Alarm", From f6bb9be7802c1e85cce6850e0ef992e19e4ac496 Mon Sep 17 00:00:00 2001 From: yuliaklochai Date: Mon, 5 May 2025 11:38:16 +0300 Subject: [PATCH 32/40] UI: trendz settings fixes --- .../pages/admin/trendz-settings.component.ts | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/admin/trendz-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/trendz-settings.component.ts index ffd2898d2a..c7f5846063 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/trendz-settings.component.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/trendz-settings.component.ts @@ -14,15 +14,14 @@ /// limitations under the License. /// -import { Component, OnInit } from '@angular/core'; +import { Component, OnInit, DestroyRef } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; -import { select, Store } from '@ngrx/store'; -import { AppState } from '@core/core.state'; import { FormBuilder, FormGroup, Validators } from '@angular/forms'; import { TrendzSettingsService } from '@core/http/trendz-settings.service'; import { TrendzSettings } from '@shared/models/trendz-settings.models'; import { isDefinedAndNotNull } from '@core/utils'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; @Component({ selector: 'tb-trendz-settings', @@ -33,10 +32,10 @@ export class TrendzSettingsComponent extends PageComponent implements OnInit, Ha trendzSettingsForm: FormGroup; - constructor(protected store: Store, - private fb: FormBuilder, - private trendzSettingsService: TrendzSettingsService) { - super(store); + constructor(private fb: FormBuilder, + private trendzSettingsService: TrendzSettingsService, + private destroyRef: DestroyRef) { + super(); } ngOnInit() { @@ -50,26 +49,26 @@ export class TrendzSettingsComponent extends PageComponent implements OnInit, Ha }); this.trendzSettingsForm.get('isTrendzEnabled').valueChanges + .pipe(takeUntilDestroyed(this.destroyRef)) .subscribe((enabled: boolean) => this.toggleUrlRequired(enabled)); } toggleUrlRequired(enabled: boolean) { const trendzUrlControl = this.trendzSettingsForm.get('trendzUrl')!; - const validators = [Validators.pattern(/^(https?:\/\/)[^\s/$.?#].[^\s]*$/i)]; if (enabled) { - validators.push(Validators.required); + trendzUrlControl.addValidators(Validators.required); + } else { + trendzUrlControl.removeValidators(Validators.required); } - trendzUrlControl.setValidators(validators); trendzUrlControl.updateValueAndValidity(); } setTrendzSettings(trendzSettings: TrendzSettings) { this.trendzSettingsForm.reset({ trendzUrl: trendzSettings?.baseUrl, - isTrendzEnabled: isDefinedAndNotNull(trendzSettings?.enabled) ? - trendzSettings?.enabled : false + isTrendzEnabled: trendzSettings?.enabled ?? false }); this.toggleUrlRequired(this.trendzSettingsForm.get('isTrendzEnabled').value); From 338d81e6c1ecccbb3db7b22e8775a0170136938f Mon Sep 17 00:00:00 2001 From: yuliaklochai Date: Tue, 6 May 2025 09:27:29 +0300 Subject: [PATCH 33/40] UI: added TrendzSettingsService to Services Map --- ui-ngx/src/app/core/http/public-api.ts | 1 + ui-ngx/src/app/core/http/trendz-settings.service.ts | 10 +++++----- ui-ngx/src/app/modules/home/models/services.map.ts | 4 +++- ui-ngx/src/app/shared/models/public-api.ts | 1 + 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/ui-ngx/src/app/core/http/public-api.ts b/ui-ngx/src/app/core/http/public-api.ts index 997689d98e..c28d80d173 100644 --- a/ui-ngx/src/app/core/http/public-api.ts +++ b/ui-ngx/src/app/core/http/public-api.ts @@ -47,3 +47,4 @@ export * from './user.service'; export * from './user-settings.service'; export * from './widget.service'; export * from './usage-info.service'; +export * from './trendz-settings.service' diff --git a/ui-ngx/src/app/core/http/trendz-settings.service.ts b/ui-ngx/src/app/core/http/trendz-settings.service.ts index 4d965f920e..82f6973e25 100644 --- a/ui-ngx/src/app/core/http/trendz-settings.service.ts +++ b/ui-ngx/src/app/core/http/trendz-settings.service.ts @@ -18,7 +18,7 @@ import { Injectable } from '@angular/core'; import { Observable } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { TrendzSettings } from '@shared/models/trendz-settings.models'; -import { defaultHttpOptionsFromConfig } from '@core/http/http-utils'; +import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; @Injectable({ providedIn: 'root' @@ -29,11 +29,11 @@ export class TrendzSettingsService { private http: HttpClient ) {} - public getTrendzSettings(): Observable { - return this.http.get(`/api/trendz/settings`, defaultHttpOptionsFromConfig({ignoreLoading: true, ignoreErrors: true})) + public getTrendzSettings(config?: RequestConfig): Observable { + return this.http.get(`/api/trendz/settings`, defaultHttpOptionsFromConfig(config)) } - public saveTrendzSettings(trendzSettings: TrendzSettings): Observable { - return this.http.post(`/api/trendz/settings`, trendzSettings, defaultHttpOptionsFromConfig({ignoreLoading: true, ignoreErrors: true})) + public saveTrendzSettings(trendzSettings: TrendzSettings, config?: RequestConfig): Observable { + return this.http.post(`/api/trendz/settings`, trendzSettings, defaultHttpOptionsFromConfig(config)) } } diff --git a/ui-ngx/src/app/modules/home/models/services.map.ts b/ui-ngx/src/app/modules/home/models/services.map.ts index e4bbc15936..517b219904 100644 --- a/ui-ngx/src/app/modules/home/models/services.map.ts +++ b/ui-ngx/src/app/modules/home/models/services.map.ts @@ -52,6 +52,7 @@ import { UiSettingsService } from '@core/http/ui-settings.service'; import { UsageInfoService } from '@core/http/usage-info.service'; import { EventService } from '@core/http/event.service'; import { AuditLogService } from '@core/http/audit-log.service'; +import { TrendzSettingsService } from '@core/http/trendz-settings.service'; export const ServicesMap = new Map>( [ @@ -91,6 +92,7 @@ export const ServicesMap = new Map>( ['usageInfoService', UsageInfoService], ['notificationService', NotificationService], ['eventService', EventService], - ['auditLogService', AuditLogService] + ['auditLogService', AuditLogService], + ['trendzSettingsService', TrendzSettingsService] ] ); diff --git a/ui-ngx/src/app/shared/models/public-api.ts b/ui-ngx/src/app/shared/models/public-api.ts index 53e4bb286e..9f7470523e 100644 --- a/ui-ngx/src/app/shared/models/public-api.ts +++ b/ui-ngx/src/app/shared/models/public-api.ts @@ -62,3 +62,4 @@ export * from './window-message.model'; export * from './usage.models'; export * from './query/query.models'; export * from './regex.constants'; +export * from './trendz-settings.models' From a0a77c721f5081e8bbfab4eed6321942290a7f60 Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Tue, 6 May 2025 13:31:34 +0200 Subject: [PATCH 34/40] fixed 'value too long' after saving ota (#13277) * fixed 'value too long' after saving ota * fixed tests --- .../server/service/entitiy/ota/DefaultTbOtaPackageService.java | 2 +- .../thingsboard/server/controller/OtaPackageControllerTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java index fca015671e..af8bbeb669 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java @@ -86,7 +86,7 @@ public class DefaultTbOtaPackageService extends AbstractTbEntityService implemen otaPackage.setContentType(contentType); otaPackage.setData(ByteBuffer.wrap(data)); otaPackage.setDataSize((long) data.length); - OtaPackageInfo savedOtaPackage = otaPackageService.saveOtaPackage(otaPackage); + OtaPackageInfo savedOtaPackage = new OtaPackageInfo(otaPackageService.saveOtaPackage(otaPackage)); logEntityActionService.logEntityAction(tenantId, savedOtaPackage.getId(), savedOtaPackage, null, actionType, user); return savedOtaPackage; } catch (Exception e) { diff --git a/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java index e8bb65dd49..3fc839996c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java @@ -216,7 +216,7 @@ public class OtaPackageControllerTest extends AbstractControllerTest { Assert.assertEquals(CHECKSUM_ALGORITHM, savedFirmware.getChecksumAlgorithm().name()); Assert.assertEquals(CHECKSUM, savedFirmware.getChecksum()); - testNotifyEntityAllOneTime(savedFirmware, savedFirmware.getId(), savedFirmware.getId(), + testNotifyEntityAllOneTime(new OtaPackageInfo(savedFirmware), savedFirmware.getId(), savedFirmware.getId(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.UPDATED); } From b74d88284c0099998fa98a9e71bb2064acd00c3b Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Tue, 6 May 2025 15:37:11 +0300 Subject: [PATCH 35/40] Add TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE to ymls --- edqs/src/main/resources/edqs.yml | 6 +++++- msa/vc-executor/src/main/resources/tb-vc-executor.yml | 5 +++++ transport/coap/src/main/resources/tb-coap-transport.yml | 5 +++++ transport/http/src/main/resources/tb-http-transport.yml | 5 +++++ transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml | 5 +++++ transport/mqtt/src/main/resources/tb-mqtt-transport.yml | 5 +++++ transport/snmp/src/main/resources/tb-snmp-transport.yml | 5 +++++ 7 files changed, 35 insertions(+), 1 deletion(-) diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index 1cc32a4230..6e6e975d68 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -148,7 +148,11 @@ queue: - key: max.poll.records # Max poll records for edqs.state topic value: "${TB_QUEUE_KAFKA_EDQS_STATE_MAX_POLL_RECORDS:512}" - + # If you override any default Kafka topic name using environment variables, you must also specify the related consumer properties + # for the new topic in `consumer-properties-per-topic-inline`. Otherwise, the topic will not inherit its expected configuration (e.g., max.poll.records, timeouts, etc). + # Format: "topic1:key1=value1,key2=value2;topic2:key=value" + # Example: "tb_core_modified.notifications:max.poll.records=10;tb_edge_modified:max.poll.records=10,enable.auto.commit=true" + consumer-properties-per-topic-inline: "${TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE:}" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index bb7f607ca0..7d1166e512 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -124,6 +124,11 @@ queue: # tb_rule_engine.sq: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" + # If you override any default Kafka topic name using environment variables, you must also specify the related consumer properties + # for the new topic in `consumer-properties-per-topic-inline`. Otherwise, the topic will not inherit its expected configuration (e.g., max.poll.records, timeouts, etc). + # Format: "topic1:key1=value1,key2=value2;topic2:key=value" + # Example: "tb_core_modified.notifications:max.poll.records=10;tb_edge_modified:max.poll.records=10,enable.auto.commit=true" + consumer-properties-per-topic-inline: "${TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE:}" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index f60a6bd47e..0c5d160625 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -310,6 +310,11 @@ queue: sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" # Protocol used to communicate with brokers. Valid values are: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + # If you override any default Kafka topic name using environment variables, you must also specify the related consumer properties + # for the new topic in `consumer-properties-per-topic-inline`. Otherwise, the topic will not inherit its expected configuration (e.g., max.poll.records, timeouts, etc). + # Format: "topic1:key1=value1,key2=value2;topic2:key=value" + # Example: "tb_core_modified.notifications:max.poll.records=10;tb_edge_modified:max.poll.records=10,enable.auto.commit=true" + consumer-properties-per-topic-inline: "${TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE:}" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index c921f9f9ae..81f4719e45 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -259,6 +259,11 @@ queue: sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" # Protocol used to communicate with brokers. Valid values are: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + # If you override any default Kafka topic name using environment variables, you must also specify the related consumer properties + # for the new topic in `consumer-properties-per-topic-inline`. Otherwise, the topic will not inherit its expected configuration (e.g., max.poll.records, timeouts, etc). + # Format: "topic1:key1=value1,key2=value2;topic2:key=value" + # Example: "tb_core_modified.notifications:max.poll.records=10;tb_edge_modified:max.poll.records=10,enable.auto.commit=true" + consumer-properties-per-topic-inline: "${TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE:}" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index 85e865e60e..2c62a35dbb 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -360,6 +360,11 @@ queue: sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" # Protocol used to communicate with brokers. Valid values are: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + # If you override any default Kafka topic name using environment variables, you must also specify the related consumer properties + # for the new topic in `consumer-properties-per-topic-inline`. Otherwise, the topic will not inherit its expected configuration (e.g., max.poll.records, timeouts, etc). + # Format: "topic1:key1=value1,key2=value2;topic2:key=value" + # Example: "tb_core_modified.notifications:max.poll.records=10;tb_edge_modified:max.poll.records=10,enable.auto.commit=true" + consumer-properties-per-topic-inline: "${TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE:}" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index a6ca2f1a6e..51f9a17005 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -293,6 +293,11 @@ queue: sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" # Protocol used to communicate with brokers. Valid values are: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + # If you override any default Kafka topic name using environment variables, you must also specify the related consumer properties + # for the new topic in `consumer-properties-per-topic-inline`. Otherwise, the topic will not inherit its expected configuration (e.g., max.poll.records, timeouts, etc). + # Format: "topic1:key1=value1,key2=value2;topic2:key=value" + # Example: "tb_core_modified.notifications:max.poll.records=10;tb_edge_modified:max.poll.records=10,enable.auto.commit=true" + consumer-properties-per-topic-inline: "${TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE:}" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 6848e8af26..f4811b3326 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -239,6 +239,11 @@ queue: sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" # Protocol used to communicate with brokers. Valid values are: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + # If you override any default Kafka topic name using environment variables, you must also specify the related consumer properties + # for the new topic in `consumer-properties-per-topic-inline`. Otherwise, the topic will not inherit its expected configuration (e.g., max.poll.records, timeouts, etc). + # Format: "topic1:key1=value1,key2=value2;topic2:key=value" + # Example: "tb_core_modified.notifications:max.poll.records=10;tb_edge_modified:max.poll.records=10,enable.auto.commit=true" + consumer-properties-per-topic-inline: "${TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE:}" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms From 381e976d87c252722e488dfd19e0416721823794 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 7 May 2025 12:17:24 +0300 Subject: [PATCH 36/40] Refactoring for input node relations --- .../update/DefaultDataUpdateService.java | 69 ++++----- .../server/dao/rule/BaseRuleChainService.java | 39 +++-- .../dao/service/RuleChainServiceTest.java | 134 ++++++++++-------- 3 files changed, 117 insertions(+), 125 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index 4c90258330..d8ddd7cd9f 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -24,21 +24,18 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.query.DynamicValue; import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.rule.RuleChainService; -import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.component.RuleNodeClassInfo; import org.thingsboard.server.service.install.DbUpgradeExecutorService; @@ -46,6 +43,7 @@ import org.thingsboard.server.utils.TbNodeUpgradeUtils; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutionException; @@ -65,9 +63,6 @@ public class DefaultDataUpdateService implements DataUpdateService { @Autowired private RelationService relationService; - @Autowired - private TenantService tenantService; - @Autowired private ComponentDiscoveryService componentDiscoveryService; @@ -78,52 +73,36 @@ public class DefaultDataUpdateService implements DataUpdateService { public void updateData() throws Exception { log.info("Updating data ..."); //TODO: should be cleaned after each release - inputNodesUpdater.updateEntities(); + updateInputNodes(); log.info("Data updated."); } - //TODO: should be removed after release - private final PaginatedUpdater inputNodesUpdater = new PaginatedUpdater<>() { - @Override - protected String getName() { - return "Input nodes updater"; - } - - @Override - protected PageData findEntities(String type, PageLink pageLink) { - return tenantService.findTenants(pageLink); - } - - @Override - protected void updateEntity(Tenant tenant) { - TenantId tenantId = tenant.getId(); + private void updateInputNodes() { + log.info("Creating relations for input nodes..."); + int n = 0; + var inputNodes = new PageDataIterable<>(pageLink -> ruleChainService.findAllRuleNodesByType(TB_RULE_CHAIN_INPUT_NODE, pageLink), 1024); + for (RuleNode inputNode : inputNodes) { try { - var inputNodes = ruleChainService.findRuleNodesByTenantIdAndType(tenantId, TB_RULE_CHAIN_INPUT_NODE); - var resultFutures = inputNodes.stream().map(ruleNode -> { - try { - JsonNode id = ruleNode.getConfiguration().get("ruleChainId"); - if (id != null) { - RuleChainId toRuleChainId = new RuleChainId(UUID.fromString(id.asText())); - RuleChainId fromRuleChainId = ruleNode.getRuleChainId(); - EntityRelation relation = new EntityRelation(); - relation.setFrom(fromRuleChainId); - relation.setTo(toRuleChainId); - relation.setType(EntityRelation.USES_TYPE); - relation.setTypeGroup(RelationTypeGroup.COMMON); - return relationService.saveRelationAsync(tenantId, relation); - } - } catch (Exception e) { - log.error("[{}] Failed to save relation for input node: [{}]", tenantId, ruleNode, e); - } - return Futures.immediateFuture(null); - }).toList(); + RuleChainId targetRuleChainId = Optional.ofNullable(inputNode.getConfiguration().get("ruleChainId")) + .filter(JsonNode::isTextual).map(JsonNode::asText).map(id -> new RuleChainId(UUID.fromString(id))) + .orElse(null); + if (targetRuleChainId == null) { + continue; + } - Futures.allAsList(resultFutures).get(); + EntityRelation relation = new EntityRelation(); + relation.setFrom(inputNode.getRuleChainId()); + relation.setTo(targetRuleChainId); + relation.setType(EntityRelation.USES_TYPE); + relation.setTypeGroup(RelationTypeGroup.COMMON); + relationService.saveRelation(TenantId.SYS_TENANT_ID, relation); + n++; } catch (Exception e) { - log.error("[{}] Unable to update Tenant input nodes", tenantId, e); + log.error("Failed to save relation for input node: {}", inputNode, e); } } - }; + log.info("Created {} relations for input nodes", n); + } @Override public void upgradeRuleNodes() { diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 1ce9bd555b..8538bd9492 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -211,9 +211,8 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC for (RuleNode existingNode : existingRuleNodes) { relationService.deleteEntityRelations(tenantId, existingNode.getId()); if (existingNode.getType().equals(TB_RULE_CHAIN_INPUT_NODE)) { - if (existingNode.getConfiguration().has("ruleChainId")) { - RuleChainId targetRuleChainId = extractRuleChainIdFromInputNode(existingNode); - var relation = createRuleChainInputRelation(ruleChainId, targetRuleChainId); + EntityRelation relation = getRuleChainInputRelation(ruleChainId, existingNode); + if (relation != null) { relationService.deleteRelation(tenantId, relation); } } @@ -242,9 +241,8 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC relations.add(new EntityRelation(ruleChainMetaData.getRuleChainId(), savedNode.getId(), EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN)); if (node.getType().equals(TB_RULE_CHAIN_INPUT_NODE)) { - if (node.getConfiguration().has("ruleChainId")) { - RuleChainId targetRuleChainId = extractRuleChainIdFromInputNode(node); - var relation = createRuleChainInputRelation(ruleChainId, targetRuleChainId); + EntityRelation relation = getRuleChainInputRelation(ruleChainId, node); + if (relation != null) { relations.add(relation); } } @@ -262,7 +260,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC firstRuleNodeId = nodes.get(ruleChainMetaData.getFirstNodeIndex()).getId(); } if ((ruleChain.getFirstRuleNodeId() != null && !ruleChain.getFirstRuleNodeId().equals(firstRuleNodeId)) - || (ruleChain.getFirstRuleNodeId() == null && firstRuleNodeId != null)) { + || (ruleChain.getFirstRuleNodeId() == null && firstRuleNodeId != null)) { ruleChain.setFirstRuleNodeId(firstRuleNodeId); } if (ruleChainMetaData.getConnections() != null) { @@ -317,19 +315,20 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC return RuleChainUpdateResult.successful(updatedRuleNodes); } - private EntityRelation createRuleChainInputRelation(RuleChainId ruleChainId, RuleChainId targetRuleChainId) { - EntityRelation relation = new EntityRelation(); - relation.setFrom(ruleChainId); - relation.setTo(targetRuleChainId); - relation.setType(EntityRelation.USES_TYPE); - relation.setTypeGroup(RelationTypeGroup.COMMON); - return relation; - } - - private RuleChainId extractRuleChainIdFromInputNode(RuleNode node) { - JsonNode configuration = node.getConfiguration(); - UUID targetUuid = UUID.fromString(configuration.get("ruleChainId").asText()); - return new RuleChainId(targetUuid); + private EntityRelation getRuleChainInputRelation(RuleChainId ruleChainId, RuleNode inputNode) { + RuleChainId targetRuleChainId = Optional.ofNullable(inputNode.getConfiguration().get("ruleChainId")) + .filter(JsonNode::isTextual).map(JsonNode::asText).map(id -> new RuleChainId(UUID.fromString(id))) + .orElse(null); + if (targetRuleChainId != null) { + EntityRelation relation = new EntityRelation(); + relation.setFrom(ruleChainId); + relation.setTo(targetRuleChainId); + relation.setType(EntityRelation.USES_TYPE); + relation.setTypeGroup(RelationTypeGroup.COMMON); + return relation; + } else { + return null; + } } @Override diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/RuleChainServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/RuleChainServiceTest.java index a2ca663619..11fa38dc19 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/RuleChainServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/RuleChainServiceTest.java @@ -16,7 +16,6 @@ package org.thingsboard.server.dao.service; import com.datastax.oss.driver.api.core.uuid.Uuids; -import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.Assert; import org.junit.Test; import org.junit.jupiter.api.Assertions; @@ -46,6 +45,7 @@ import java.util.List; import java.util.UUID; import java.util.function.Function; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.thingsboard.server.common.data.relation.EntityRelation.USES_TYPE; import static org.thingsboard.server.dao.rule.BaseRuleChainService.TB_RULE_CHAIN_INPUT_NODE; @@ -277,7 +277,7 @@ public class RuleChainServiceTest extends AbstractServiceTest { List ruleNodes = savedRuleChainMetaData.getNodes(); int name3Index = -1; - for (int i=0;i ruleNodes = new ArrayList<>(); - ruleNodes.add(ruleNode); + ruleNodes.add(toRuleChain1Node); + ruleNodes.add(toRuleChain1Node2); ruleChainMetaData.setFirstNodeIndex(0); ruleChainMetaData.setNodes(ruleNodes); ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData, Function.identity()); - List relations = relationService.findByFromAndType(tenantId, savedFromRuleChain.getId(), USES_TYPE, RelationTypeGroup.COMMON); - Assert.assertEquals(1, relations.size()); - EntityRelation usesRelation = relations.get(0); - Assert.assertEquals(savedFromRuleChain.getId(), usesRelation.getFrom()); - Assert.assertEquals(savedToRuleChain.getId(), usesRelation.getTo()); + List relations = relationService.findByFromAndType(tenantId, fromRuleChainId, USES_TYPE, RelationTypeGroup.COMMON); + assertThat(relations).singleElement().satisfies(relationToRuleChain1 -> { + assertThat(relationToRuleChain1.getFrom()).isEqualTo(fromRuleChainId); + assertThat(relationToRuleChain1.getTo()).isEqualTo(toRuleChain1Id); + }); - RuleChain newToRuleChain = new RuleChain(); - newToRuleChain.setName("New To Rule Chain"); - newToRuleChain.setTenantId(tenantId); - RuleChain savedNewToRuleChain = ruleChainService.saveRuleChain(newToRuleChain); + RuleChain toRuleChain2 = new RuleChain(); + toRuleChain2.setName("To Rule Chain 2"); + toRuleChain2.setTenantId(tenantId); + toRuleChain2 = ruleChainService.saveRuleChain(toRuleChain2); + RuleChainId toRuleChain2Id = toRuleChain2.getId(); - RuleNode newRuleNode = new RuleNode(); - newRuleNode.setName("Input node"); - newRuleNode.setType(TB_RULE_CHAIN_INPUT_NODE); - ObjectNode newConfiguration = JacksonUtil.newObjectNode(); - configuration.put("ruleChainId", savedNewToRuleChain.getId().toString()); - newRuleNode.setConfiguration(newConfiguration); + RuleNode toRuleChain2Node = new RuleNode(); + toRuleChain2Node.setName("To Rule Chain 2"); + toRuleChain2Node.setType(TB_RULE_CHAIN_INPUT_NODE); + toRuleChain2Node.setConfiguration(JacksonUtil.newObjectNode() + .put("ruleChainId", toRuleChain2Id.toString())); List newRuleNodes = new ArrayList<>(); - newRuleNodes.add(newRuleNode); - RuleChainMetaData foundRuleChainMetaData = ruleChainService.loadRuleChainMetaData(tenantId, ruleChainMetaData.getRuleChainId()); - foundRuleChainMetaData.setNodes(newRuleNodes); + newRuleNodes.add(toRuleChain2Node); + newRuleNodes.add(toRuleChain1Node); + ruleChainMetaData = ruleChainService.loadRuleChainMetaData(tenantId, ruleChainMetaData.getRuleChainId()); + ruleChainMetaData.setNodes(newRuleNodes); ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData, Function.identity()); - List newRelations = relationService.findByFromAndType(tenantId, savedFromRuleChain.getId(), USES_TYPE, RelationTypeGroup.COMMON); - Assert.assertEquals(1, relations.size()); - EntityRelation newUsesRelation = newRelations.get(0); - Assert.assertEquals(savedFromRuleChain.getId(), newUsesRelation.getFrom()); - Assert.assertEquals(savedNewToRuleChain.getId(), newUsesRelation.getTo()); + List newRelations = relationService.findByFromAndType(tenantId, fromRuleChainId, USES_TYPE, RelationTypeGroup.COMMON); + assertThat(newRelations).hasSize(2); + assertThat(newRelations).anySatisfy(relationToRuleChain1 -> { + assertThat(relationToRuleChain1.getFrom()).isEqualTo(fromRuleChainId); + assertThat(relationToRuleChain1.getTo()).isEqualTo(toRuleChain1Id); + }); + assertThat(newRelations).anySatisfy(relationToRuleChain2 -> { + assertThat(relationToRuleChain2.getFrom()).isEqualTo(fromRuleChainId); + assertThat(relationToRuleChain2.getTo()).isEqualTo(toRuleChain2Id); + }); } private RuleChainId saveRuleChainAndSetAutoAssignToEdge(String name) { @@ -462,9 +476,9 @@ public class RuleChainServiceTest extends AbstractServiceTest { ruleChainMetaData.setFirstNodeIndex(0); ruleChainMetaData.setNodes(ruleNodes); - ruleChainMetaData.addConnectionInfo(0,1,"success"); - ruleChainMetaData.addConnectionInfo(0,2,"fail"); - ruleChainMetaData.addConnectionInfo(1,2,"success"); + ruleChainMetaData.addConnectionInfo(0, 1, "success"); + ruleChainMetaData.addConnectionInfo(0, 2, "fail"); + ruleChainMetaData.addConnectionInfo(1, 2, "success"); Assert.assertTrue(ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData, Function.identity()).isSuccess()); return ruleChainService.loadRuleChainMetaData(tenantId, ruleChainMetaData.getRuleChainId()); @@ -501,10 +515,10 @@ public class RuleChainServiceTest extends AbstractServiceTest { ruleChainMetaData.setFirstNodeIndex(0); ruleChainMetaData.setNodes(ruleNodes); - ruleChainMetaData.addConnectionInfo(0,1,"success"); - ruleChainMetaData.addConnectionInfo(0,2,"fail"); - ruleChainMetaData.addConnectionInfo(1,2,"success"); - ruleChainMetaData.addConnectionInfo(2,2,"success"); + ruleChainMetaData.addConnectionInfo(0, 1, "success"); + ruleChainMetaData.addConnectionInfo(0, 2, "fail"); + ruleChainMetaData.addConnectionInfo(1, 2, "success"); + ruleChainMetaData.addConnectionInfo(2, 2, "success"); return ruleChainMetaData; } @@ -540,10 +554,10 @@ public class RuleChainServiceTest extends AbstractServiceTest { ruleChainMetaData.setFirstNodeIndex(0); ruleChainMetaData.setNodes(ruleNodes); - ruleChainMetaData.addConnectionInfo(0,1,"success"); - ruleChainMetaData.addConnectionInfo(0,2,"fail"); - ruleChainMetaData.addConnectionInfo(1,2,"success"); - ruleChainMetaData.addConnectionInfo(2,0,"success"); + ruleChainMetaData.addConnectionInfo(0, 1, "success"); + ruleChainMetaData.addConnectionInfo(0, 2, "fail"); + ruleChainMetaData.addConnectionInfo(1, 2, "success"); + ruleChainMetaData.addConnectionInfo(2, 0, "success"); return ruleChainMetaData; } @@ -649,16 +663,16 @@ public class RuleChainServiceTest extends AbstractServiceTest { private RuleChain getRuleChain() { String ruleChainStr = "{\n" + - " \"name\": \"Root Rule Chain\",\n" + - " \"type\": \"CORE\",\n" + - " \"firstRuleNodeId\": {\n" + - " \"entityType\": \"RULE_NODE\",\n" + - " \"id\": \"91ad0b00-e779-11ee-9cf0-15d8b6079fdb\"\n" + - " },\n" + - " \"debugMode\": false,\n" + - " \"configuration\": null,\n" + - " \"additionalInfo\": null\n" + - "}"; + " \"name\": \"Root Rule Chain\",\n" + + " \"type\": \"CORE\",\n" + + " \"firstRuleNodeId\": {\n" + + " \"entityType\": \"RULE_NODE\",\n" + + " \"id\": \"91ad0b00-e779-11ee-9cf0-15d8b6079fdb\"\n" + + " },\n" + + " \"debugMode\": false,\n" + + " \"configuration\": null,\n" + + " \"additionalInfo\": null\n" + + "}"; return JacksonUtil.fromString(ruleChainStr, RuleChain.class); } } From 1d83a2c9d387e4b629adbc1b8156f62f0dc6b171 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 7 May 2025 13:01:09 +0300 Subject: [PATCH 37/40] Refactoring for TbMsgProto usage --- ...riginatorIdTbRuleEngineSubmitStrategy.java | 8 +--- ...TbRuleEngineProcessingStrategyFactory.java | 8 ++-- .../TbRuleEngineQueueConsumerManager.java | 15 +++---- .../thingsboard/server/common/msg/TbMsg.java | 45 +++++++++---------- common/message/src/main/proto/tbmsg.proto | 4 +- .../server/common/util/ProtoUtils.java | 22 ++++++++- common/proto/src/main/proto/queue.proto | 2 +- 7 files changed, 55 insertions(+), 49 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java index aec9252356..cec2aa19da 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java @@ -20,6 +20,7 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.msg.gen.MsgProtos; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.UUID; @@ -34,12 +35,7 @@ public class SequentialByOriginatorIdTbRuleEngineSubmitStrategy extends Sequenti @Override protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { try { - MsgProtos.TbMsgProto proto; - if (msg.getTbMsg().isEmpty()) { - proto = msg.getTbMsgProto(); - } else { - proto = MsgProtos.TbMsgProto.parseFrom(msg.getTbMsg()); - } + MsgProtos.TbMsgProto proto = ProtoUtils.getTbMsgProto(msg); return EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); } catch (InvalidProtocolBufferException e) { log.warn("[{}] Failed to parse TbMsg: {}", queueName, msg); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java index 2adcd9b199..023b4d4cc7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -19,8 +19,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; import org.thingsboard.server.common.data.queue.ProcessingStrategy; -import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -125,7 +125,7 @@ public class TbRuleEngineProcessingStrategyFactory { } log.debug("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); if (log.isTraceEnabled()) { - toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromProto(result.getQueueName(), msg.getValue().getTbMsgProto(), msg.getValue().getTbMsg(), TbMsgCallback.EMPTY))); + toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, ProtoUtils.fromTbMsgProto(result.getQueueName(), msg.getValue(), TbMsgCallback.EMPTY))); } if (pauseBetweenRetries > 0) { try { @@ -164,10 +164,10 @@ public class TbRuleEngineProcessingStrategyFactory { log.debug("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); } if (log.isTraceEnabled()) { - result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromProto(result.getQueueName(), msg.getValue().getTbMsgProto(), msg.getValue().getTbMsg(), TbMsgCallback.EMPTY))); + result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, ProtoUtils.fromTbMsgProto(result.getQueueName(), msg.getValue(), TbMsgCallback.EMPTY))); } if (log.isTraceEnabled()) { - result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, TbMsg.fromProto(result.getQueueName(), msg.getValue().getTbMsgProto(), msg.getValue().getTbMsg(), TbMsgCallback.EMPTY))); + result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, ProtoUtils.fromTbMsgProto(result.getQueueName(), msg.getValue(), TbMsgCallback.EMPTY))); } return new TbRuleEngineProcessingDecision(true, null); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index 01368f780e..3aa0d2eabd 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.msg.queue.RuleNodeInfo; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -178,7 +179,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager relationTypes; @@ -207,7 +208,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager> pending : map.entrySet()) { ToRuleEngineMsg tmp = pending.getValue().getValue(); - TbMsg tmpMsg = TbMsg.fromProto(config.getName(), tmp.getTbMsgProto(), tmp.getTbMsg(), TbMsgCallback.EMPTY); + TbMsg tmpMsg = ProtoUtils.fromTbMsgProto(config.getName(), tmp, TbMsgCallback.EMPTY); RuleNodeInfo ruleNodeInfo = ctx.getLastVisitedRuleNode(pending.getKey()); if (printAll) { log.trace("[{}][{}] {} to process message: {}, Last Rule Node: {}", queueKey, TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); @@ -236,13 +237,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager msg : msgs) { try { - MsgProtos.TbMsgProto tbMsgProto; - if (msg.getValue().getTbMsg().isEmpty()) { - tbMsgProto = msg.getValue().getTbMsgProto(); - } else { - tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg()); - } - + MsgProtos.TbMsgProto tbMsgProto = ProtoUtils.getTbMsgProto(msg.getValue()); EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); TopicPartitionInfo tpi = ctx.getPartitionService().resolve(ServiceType.TB_RULE_ENGINE, config.getName(), TenantId.SYS_TENANT_ID, originator); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index b7447b14ba..1d8d9497a9 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.msg.gen.MsgProtos; +import org.thingsboard.server.common.msg.gen.MsgProtos.TbMsgProto; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; @@ -151,8 +152,8 @@ public final class TbMsg implements Serializable { this.callback = Objects.requireNonNullElse(callback, TbMsgCallback.EMPTY); } - public static MsgProtos.TbMsgProto toProto(TbMsg msg) { - MsgProtos.TbMsgProto.Builder builder = MsgProtos.TbMsgProto.newBuilder(); + public static TbMsgProto toProto(TbMsg msg) { + TbMsgProto.Builder builder = TbMsgProto.newBuilder(); builder.setId(msg.getId().toString()); builder.setTs(msg.getTs()); builder.setType(msg.getType()); @@ -204,12 +205,11 @@ public final class TbMsg implements Serializable { return builder.build(); } - //TODO: added for processing old messages from queue, should be removed after release - @Deprecated(forRemoval = true) - public static TbMsg fromProto(String queueName, MsgProtos.TbMsgProto proto, ByteString data, TbMsgCallback callback) { + @Deprecated(forRemoval = true, since = "4.1") // to be removed in 4.2 + public static TbMsg fromProto(String queueName, TbMsgProto proto, ByteString data, TbMsgCallback callback) { try { if (!data.isEmpty()) { - proto = MsgProtos.TbMsgProto.parseFrom(data); + proto = TbMsgProto.parseFrom(data); } } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); @@ -217,7 +217,7 @@ public final class TbMsg implements Serializable { return fromProto(queueName, proto, callback); } - public static TbMsg fromProto(String queueName, MsgProtos.TbMsgProto proto, TbMsgCallback callback) { + public static TbMsg fromProto(String queueName, TbMsgProto proto, TbMsgCallback callback) { TbMsgMetaData metaData = new TbMsgMetaData(proto.getMetaData().getDataMap()); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); CustomerId customerId = null; @@ -225,7 +225,8 @@ public final class TbMsg implements Serializable { RuleNodeId ruleNodeId = null; UUID correlationId = null; Integer partition = null; - List calculatedFieldIds = new CopyOnWriteArrayList<>();if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) { + List calculatedFieldIds = new CopyOnWriteArrayList<>(); + if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) { customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB())); } if (proto.getRuleChainIdMSB() != 0L && proto.getRuleChainIdLSB() != 0L) { @@ -240,19 +241,13 @@ public final class TbMsg implements Serializable { } for (MsgProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) { - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID( - cfIdProto.getCalculatedFieldIdMSB(), - cfIdProto.getCalculatedFieldIdLSB() - )); - calculatedFieldIds.add(calculatedFieldId); - }TbMsgProcessingCtx ctx; - if (proto.hasCtx()) { - ctx = TbMsgProcessingCtx.fromProto(proto.getCtx()); - } else { - // Backward compatibility with unprocessed messages fetched from queue after update. - ctx = new TbMsgProcessingCtx(proto.getRuleNodeExecCounter()); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID( + cfIdProto.getCalculatedFieldIdMSB(), + cfIdProto.getCalculatedFieldIdLSB() + )); + calculatedFieldIds.add(calculatedFieldId); } - + TbMsgProcessingCtx ctx = TbMsgProcessingCtx.fromProto(proto.getCtx()); TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), null, proto.getType(), entityId, customerId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback); @@ -505,11 +500,11 @@ public final class TbMsg implements Serializable { public String toString() { return "TbMsg.TbMsgBuilder(queueName=" + this.queueName + ", id=" + this.id + ", ts=" + this.ts + - ", type=" + this.type + ", internalType=" + this.internalType + ", originator=" + this.originator + - ", customerId=" + this.customerId + ", metaData=" + this.metaData + ", dataType=" + this.dataType + - ", data=" + this.data + ", ruleChainId=" + this.ruleChainId + ", ruleNodeId=" + this.ruleNodeId + - ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", previousCalculatedFields=" + this.previousCalculatedFieldIds + - ", ctx=" + this.ctx + ", callback=" + this.callback + ")"; + ", type=" + this.type + ", internalType=" + this.internalType + ", originator=" + this.originator + + ", customerId=" + this.customerId + ", metaData=" + this.metaData + ", dataType=" + this.dataType + + ", data=" + this.data + ", ruleChainId=" + this.ruleChainId + ", ruleNodeId=" + this.ruleNodeId + + ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", previousCalculatedFields=" + this.previousCalculatedFieldIds + + ", ctx=" + this.ctx + ", callback=" + this.callback + ")"; } } diff --git a/common/message/src/main/proto/tbmsg.proto b/common/message/src/main/proto/tbmsg.proto index 65a967e9e4..e70104d503 100644 --- a/common/message/src/main/proto/tbmsg.proto +++ b/common/message/src/main/proto/tbmsg.proto @@ -59,8 +59,8 @@ message TbMsgProto { string data = 14; int64 ts = 15; - // Will be removed in 3.4. Moved to processing context - int32 ruleNodeExecCounter = 16; + + // ruleNodeExecCounter (16) was removed in 4.1 int64 customerIdMSB = 17; int64 customerIdLSB = 18; diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 1ebd753f3c..502884eb31 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -18,6 +18,8 @@ package org.thingsboard.server.common.util; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.ApiUsageRecordKey; @@ -76,12 +78,15 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; import org.thingsboard.server.common.msg.edge.EdgeHighPriorityMsg; import org.thingsboard.server.common.msg.edge.FromEdgeSyncResponse; import org.thingsboard.server.common.msg.edge.ToEdgeSyncRequest; +import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponseActorMsg; import org.thingsboard.server.common.msg.rpc.RemoveRpcActorMsg; @@ -93,8 +98,8 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceDeleteMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; import org.thingsboard.server.gen.transport.TransportProtos.ApiUsageRecordKeyProto; +import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; import java.util.ArrayList; import java.util.Arrays; @@ -1344,6 +1349,21 @@ public class ProtoUtils { return builder.build(); } + @Deprecated(forRemoval = true, since = "4.1") + public static MsgProtos.TbMsgProto getTbMsgProto(TransportProtos.ToRuleEngineMsg ruleEngineMsg) throws InvalidProtocolBufferException { + if (ruleEngineMsg.getTbMsg().isEmpty()) { + return ruleEngineMsg.getTbMsgProto(); + } else { + return MsgProtos.TbMsgProto.parseFrom(ruleEngineMsg.getTbMsg()); + } + } + + @SneakyThrows + @Deprecated(forRemoval = true, since = "4.1") // inline to TbMsg.fromProto(queueName, ruleEngineMsg.getTbMsgProto(), callback) + public static TbMsg fromTbMsgProto(String queueName, TransportProtos.ToRuleEngineMsg ruleEngineMsg, TbMsgCallback callback) { + return TbMsg.fromProto(queueName, getTbMsgProto(ruleEngineMsg), callback); + } + private static boolean isNotNull(Object obj) { return obj != null; } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 449f662a93..2a97fd35d0 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1703,7 +1703,7 @@ message ToCalculatedFieldNotificationMsg { message ToRuleEngineMsg { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; - bytes tbMsg = 3 [deprecated = true]; + bytes tbMsg = 3 [deprecated = true]; // for removal in 4.2 repeated string relationTypes = 4; string failureMessage = 5; msgqueue.TbMsgProto tbMsgProto = 6; From 32212b9c5129d5b60a968310635fd98179118fee Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 12 May 2025 12:08:52 +0300 Subject: [PATCH 38/40] Monitoring: automatic rule chain update --- .../ThingsboardMonitoringApplication.java | 6 + .../monitoring/client/TbClient.java | 9 +- .../monitoring/service/BaseHealthChecker.java | 5 +- .../service/BaseMonitoringService.java | 10 +- .../service/MonitoringEntityService.java | 245 +++++++++++++ .../service/MonitoringReporter.java | 14 +- .../transport/TransportHealthChecker.java | 149 +------- .../monitoring/util/ResourceUtils.java | 16 +- .../{root_rule_chain.json => rule_chain.json} | 327 ++++++++---------- 9 files changed, 428 insertions(+), 353 deletions(-) create mode 100644 monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringEntityService.java rename monitoring/src/main/resources/{root_rule_chain.json => rule_chain.json} (76%) diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/ThingsboardMonitoringApplication.java b/monitoring/src/main/java/org/thingsboard/monitoring/ThingsboardMonitoringApplication.java index f5f9e73275..c8e8d7070b 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/ThingsboardMonitoringApplication.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/ThingsboardMonitoringApplication.java @@ -25,6 +25,7 @@ import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.EnableScheduling; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.monitoring.service.BaseMonitoringService; +import org.thingsboard.monitoring.service.MonitoringEntityService; import java.util.List; import java.util.Map; @@ -38,6 +39,8 @@ public class ThingsboardMonitoringApplication { @Autowired private List> monitoringServices; + @Autowired + private MonitoringEntityService entityService; @Value("${monitoring.monitoring_rate_ms}") private int monitoringRateMs; @@ -50,6 +53,9 @@ public class ThingsboardMonitoringApplication { @EventListener(ApplicationReadyEvent.class) public void startMonitoring() { + entityService.checkEntities(); + monitoringServices.forEach(BaseMonitoringService::init); + ScheduledExecutorService scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("monitoring-executor"); scheduler.scheduleWithFixedDelay(() -> { monitoringServices.forEach(monitoringService -> { diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/client/TbClient.java b/monitoring/src/main/java/org/thingsboard/monitoring/client/TbClient.java index e6a7e1b8af..0884cf7e5c 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/client/TbClient.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/client/TbClient.java @@ -15,17 +15,15 @@ */ package org.thingsboard.monitoring.client; +import jakarta.annotation.PostConstruct; import org.springframework.beans.factory.annotation.Value; -import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.boot.web.client.RestTemplateBuilder; -import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; import org.thingsboard.rest.client.RestClient; import java.time.Duration; @Component -@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) public class TbClient extends RestClient { @Value("${monitoring.rest.username}") @@ -41,6 +39,11 @@ public class TbClient extends RestClient { .build(), baseUrl); } + @PostConstruct + private void init() { + logIn(); + } + public String logIn() { login(username, password); return getToken(); diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java index 1e9cdbe191..d19e723332 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java @@ -22,7 +22,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.thingsboard.monitoring.client.TbClient; import org.thingsboard.monitoring.client.WsClient; import org.thingsboard.monitoring.config.MonitoringConfig; import org.thingsboard.monitoring.config.MonitoringTarget; @@ -46,6 +45,8 @@ public abstract class BaseHealthChecker, T ext @Value("${monitoring.calculated_fields.enabled:true}") protected boolean checkCalculatedFields; - @PostConstruct - private void init() { + public void init() { if (configs == null || configs.isEmpty()) { return; } - tbClient.logIn(); + configs.forEach(config -> { config.getTargets().forEach(target -> { BaseHealthChecker healthChecker = initHealthChecker(target, config); @@ -108,7 +106,7 @@ public abstract class BaseMonitoringService, T ext private BaseHealthChecker initHealthChecker(T target, C config) { BaseHealthChecker healthChecker = (BaseHealthChecker) createHealthChecker(config, target); log.info("Initializing {} for {}", healthChecker.getClass().getSimpleName(), target.getBaseUrl()); - healthChecker.initialize(tbClient); + healthChecker.initialize(); devices.add(target.getDeviceId()); return healthChecker; } @@ -140,7 +138,7 @@ public abstract class BaseMonitoringService, T ext reporter.serviceIsOk(MonitoredServiceKey.EDQS); } - reporter.reportLatencies(tbClient); + reporter.reportLatencies(); log.debug("Finished {}", getName()); } catch (ServiceFailureException e) { reporter.serviceFailure(e.getServiceKey(), e); diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringEntityService.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringEntityService.java new file mode 100644 index 0000000000..66dc9200e3 --- /dev/null +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringEntityService.java @@ -0,0 +1,245 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.monitoring.service; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.RegexUtils; +import org.thingsboard.monitoring.client.TbClient; +import org.thingsboard.monitoring.config.transport.DeviceConfig; +import org.thingsboard.monitoring.config.transport.TransportMonitoringConfig; +import org.thingsboard.monitoring.config.transport.TransportMonitoringTarget; +import org.thingsboard.monitoring.config.transport.TransportType; +import org.thingsboard.monitoring.util.ResourceUtils; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MBootstrapClientCredentials; +import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MDeviceCredentials; +import org.thingsboard.server.common.data.device.credentials.lwm2m.NoSecBootstrapClientCredential; +import org.thingsboard.server.common.data.device.credentials.lwm2m.NoSecClientCredential; +import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; +import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.data.DeviceData; +import org.thingsboard.server.common.data.device.data.Lwm2mDeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; +import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.DeviceProfileData; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.common.data.security.DeviceCredentialsType; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.thingsboard.monitoring.service.BaseHealthChecker.TEST_CF_TELEMETRY_KEY; +import static org.thingsboard.monitoring.service.BaseHealthChecker.TEST_TELEMETRY_KEY; + +@Service +@Slf4j +@RequiredArgsConstructor +public class MonitoringEntityService { + + private final TbClient tbClient; + + @Value("${monitoring.calculated_fields.enabled:true}") + private boolean calculatedFieldsMonitoringEnabled; + + public void checkEntities() { + RuleChain ruleChain = tbClient.getRuleChains(RuleChainType.CORE, new PageLink(10)).getData().stream() + .filter(RuleChain::isRoot) + .findFirst().orElseThrow(); + RuleChainId ruleChainId = ruleChain.getId(); + + JsonNode ruleChainDescriptor = ResourceUtils.getResource("rule_chain.json"); + List attributeKeys = tbClient.getAttributeKeys(ruleChainId); + Map attributes = tbClient.getAttributeKvEntries(ruleChainId, attributeKeys).stream() + .collect(Collectors.toMap(KvEntry::getKey, KvEntry::getValueAsString)); + + int currentVersion = Integer.parseInt(attributes.getOrDefault("version", "0")); + int newVersion = ruleChainDescriptor.get("version").asInt(); + if (currentVersion == newVersion) { + log.info("Not updating rule chain, version is the same ({})", currentVersion); + return; + } else { + log.info("Updating rule chain '{}' from version {} to {}", ruleChain.getName(), currentVersion, newVersion); + } + + String metadataJson = RegexUtils.replace(ruleChainDescriptor.get("metadata").toString(), + "\\$\\{MONITORING:(.+?)}", matchResult -> { + String key = matchResult.group(1); + String value = attributes.get(key); + if (value == null) { + throw new IllegalArgumentException("No attribute found for key " + key); + } + log.info("Using {}: {}", key, value); + return value; + }); + RuleChainMetaData metaData = JacksonUtil.fromString(metadataJson, RuleChainMetaData.class); + metaData.setRuleChainId(ruleChainId); + tbClient.saveRuleChainMetaData(metaData); + } + + public Asset getOrCreateMonitoringAsset() { + String assetName = "[Monitoring] Latencies"; + return tbClient.findAsset(assetName).orElseGet(() -> { + Asset asset = new Asset(); + asset.setType("Monitoring"); + asset.setName(assetName); + asset = tbClient.saveAsset(asset); + log.info("Created monitoring asset {}", asset.getId()); + return asset; + }); + } + + public void checkEntities(TransportMonitoringConfig config, TransportMonitoringTarget target) { + Device device = getOrCreateDevice(config, target); + DeviceCredentials credentials = tbClient.getDeviceCredentialsByDeviceId(device.getId()) + .orElseThrow(() -> new IllegalArgumentException("No credentials found for device " + device.getId())); + + DeviceConfig deviceConfig = new DeviceConfig(); + deviceConfig.setId(device.getId().toString()); + deviceConfig.setName(device.getName()); + deviceConfig.setCredentials(credentials); + target.setDevice(deviceConfig); + } + + private Device getOrCreateDevice(TransportMonitoringConfig config, TransportMonitoringTarget target) { + TransportType transportType = config.getTransportType(); + String deviceName = String.format("%s %s (%s) - %s", target.getNamePrefix(), transportType.getName(), target.getQueue(), target.getBaseUrl()).trim(); + Device device = tbClient.getTenantDevice(deviceName).orElse(null); + if (device != null) { + if (calculatedFieldsMonitoringEnabled) { + CalculatedField calculatedField = tbClient.getCalculatedFieldsByEntityId(device.getId(), new PageLink(1, 0, TEST_CF_TELEMETRY_KEY)) + .getData().stream().findFirst().orElse(null); + if (calculatedField == null) { + createCalculatedField(device); + } + } + return device; + } + + log.info("Creating new device '{}'", deviceName); + device = new Device(); + device.setName(deviceName); + + DeviceCredentials credentials = new DeviceCredentials(); + credentials.setCredentialsId(RandomStringUtils.randomAlphabetic(20)); + DeviceData deviceData = new DeviceData(); + deviceData.setConfiguration(new DefaultDeviceConfiguration()); + + DeviceProfile deviceProfile = getOrCreateDeviceProfile(config, target); + device.setType(deviceProfile.getName()); + device.setDeviceProfileId(deviceProfile.getId()); + + if (transportType != TransportType.LWM2M) { + deviceData.setTransportConfiguration(new DefaultDeviceTransportConfiguration()); + credentials.setCredentialsType(DeviceCredentialsType.ACCESS_TOKEN); + } else { + deviceData.setTransportConfiguration(new Lwm2mDeviceTransportConfiguration()); + credentials.setCredentialsType(DeviceCredentialsType.LWM2M_CREDENTIALS); + LwM2MDeviceCredentials lwm2mCreds = new LwM2MDeviceCredentials(); + NoSecClientCredential client = new NoSecClientCredential(); + client.setEndpoint(credentials.getCredentialsId()); + lwm2mCreds.setClient(client); + LwM2MBootstrapClientCredentials bootstrap = new LwM2MBootstrapClientCredentials(); + bootstrap.setBootstrapServer(new NoSecBootstrapClientCredential()); + bootstrap.setLwm2mServer(new NoSecBootstrapClientCredential()); + lwm2mCreds.setBootstrap(bootstrap); + credentials.setCredentialsValue(JacksonUtil.toString(lwm2mCreds)); + } + + return tbClient.saveDeviceWithCredentials(device, credentials).get(); + } + + private DeviceProfile getOrCreateDeviceProfile(TransportMonitoringConfig config, TransportMonitoringTarget target) { + TransportType transportType = config.getTransportType(); + String profileName = String.format("%s %s (%s)", target.getNamePrefix(), transportType.getName(), target.getQueue()).trim(); + DeviceProfile deviceProfile = tbClient.getDeviceProfiles(new PageLink(1, 0, profileName)).getData() + .stream().findFirst().orElse(null); + if (deviceProfile != null) { + return deviceProfile; + } + + log.info("Creating new device profile '{}'", profileName); + if (transportType != TransportType.LWM2M) { + deviceProfile = new DeviceProfile(); + deviceProfile.setType(DeviceProfileType.DEFAULT); + deviceProfile.setTransportType(DeviceTransportType.DEFAULT); + DeviceProfileData profileData = new DeviceProfileData(); + profileData.setConfiguration(new DefaultDeviceProfileConfiguration()); + profileData.setTransportConfiguration(new DefaultDeviceProfileTransportConfiguration()); + deviceProfile.setProfileData(profileData); + } else { + tbClient.getResources(new PageLink(1, 0, "LwM2M Monitoring")).getData() + .stream().findFirst() + .orElseGet(() -> { + TbResource newResource = ResourceUtils.getResource("lwm2m/resource.json", TbResource.class); + log.info("Creating LwM2M resource"); + return tbClient.saveResource(newResource); + }); + deviceProfile = ResourceUtils.getResource("lwm2m/device_profile.json", DeviceProfile.class); + } + + deviceProfile.setName(profileName); + deviceProfile.setDefaultQueueName(target.getQueue()); + return tbClient.saveDeviceProfile(deviceProfile); + } + + private void createCalculatedField(Device device) { + log.info("Creating calculated field for device '{}'", device.getName()); + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setName(TEST_CF_TELEMETRY_KEY); + calculatedField.setEntityId(device.getId()); + calculatedField.setType(CalculatedFieldType.SCRIPT); + ScriptCalculatedFieldConfiguration configuration = new ScriptCalculatedFieldConfiguration(); + Argument testDataArgument = new Argument(); + testDataArgument.setRefEntityKey(new ReferencedEntityKey(TEST_TELEMETRY_KEY, ArgumentType.TS_LATEST, null)); + configuration.setArguments(Map.of( + TEST_TELEMETRY_KEY, testDataArgument + )); + configuration.setExpression("return { \"" + TEST_CF_TELEMETRY_KEY + "\": " + TEST_TELEMETRY_KEY + " + \"-cf\" };"); + Output output = new Output(); + output.setType(OutputType.TIME_SERIES); + configuration.setOutput(output); + calculatedField.setConfiguration(configuration); + calculatedField.setDebugMode(true); + tbClient.saveCalculatedField(calculatedField); + } + +} diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringReporter.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringReporter.java index 62ed0d74aa..99cb9feb3d 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringReporter.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringReporter.java @@ -46,6 +46,8 @@ import java.util.stream.Collectors; public class MonitoringReporter { private final NotificationService notificationService; + private final TbClient tbClient; + private final MonitoringEntityService entityService; private final Map latencies = new ConcurrentHashMap<>(); private final Map failuresCounters = new ConcurrentHashMap<>(); @@ -62,7 +64,7 @@ public class MonitoringReporter { @Value("${monitoring.latency.reporting_asset_id}") private String reportingAssetId; - public void reportLatencies(TbClient tbClient) { + public void reportLatencies() { if (latencies.isEmpty()) { return; } @@ -81,15 +83,7 @@ public class MonitoringReporter { try { if (StringUtils.isBlank(reportingAssetId)) { - String assetName = "[Monitoring] Latencies"; - Asset monitoringAsset = tbClient.findAsset(assetName).orElseGet(() -> { - Asset asset = new Asset(); - asset.setType("Monitoring"); - asset.setName(assetName); - asset = tbClient.saveAsset(asset); - log.info("Created monitoring asset {}", asset.getId()); - return asset; - }); + Asset monitoringAsset = entityService.getOrCreateMonitoringAsset(); reportingAssetId = monitoringAsset.getId().toString(); } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java index ea95740c68..946c3e6d3d 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java @@ -17,46 +17,13 @@ package org.thingsboard.monitoring.service.transport; import com.fasterxml.jackson.databind.node.TextNode; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.RandomStringUtils; import org.springframework.beans.factory.annotation.Value; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.monitoring.client.TbClient; -import org.thingsboard.monitoring.config.transport.DeviceConfig; import org.thingsboard.monitoring.config.transport.TransportInfo; import org.thingsboard.monitoring.config.transport.TransportMonitoringConfig; import org.thingsboard.monitoring.config.transport.TransportMonitoringTarget; import org.thingsboard.monitoring.config.transport.TransportType; import org.thingsboard.monitoring.service.BaseHealthChecker; -import org.thingsboard.monitoring.util.ResourceUtils; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.DeviceProfileType; -import org.thingsboard.server.common.data.DeviceTransportType; -import org.thingsboard.server.common.data.TbResource; -import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.cf.configuration.OutputType; -import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; -import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MBootstrapClientCredentials; -import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MDeviceCredentials; -import org.thingsboard.server.common.data.device.credentials.lwm2m.NoSecBootstrapClientCredential; -import org.thingsboard.server.common.data.device.credentials.lwm2m.NoSecClientCredential; -import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; -import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; -import org.thingsboard.server.common.data.device.data.DeviceData; -import org.thingsboard.server.common.data.device.data.Lwm2mDeviceTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; -import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.DeviceProfileData; -import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.security.DeviceCredentials; -import org.thingsboard.server.common.data.security.DeviceCredentialsType; - -import java.util.Map; @Slf4j public abstract class TransportHealthChecker extends BaseHealthChecker { @@ -69,16 +36,8 @@ public abstract class TransportHealthChecker new IllegalArgumentException("No credentials found for device " + device.getId())); - - DeviceConfig deviceConfig = new DeviceConfig(); - deviceConfig.setId(device.getId().toString()); - deviceConfig.setName(device.getName()); - deviceConfig.setCredentials(credentials); - target.setDevice(deviceConfig); + protected void initialize() { + entityService.checkEntities(config, target); } @Override @@ -98,110 +57,6 @@ public abstract class TransportHealthChecker { - TbResource newResource = ResourceUtils.getResource("lwm2m/resource.json", TbResource.class); - log.info("Creating LwM2M resource"); - return tbClient.saveResource(newResource); - }); - deviceProfile = ResourceUtils.getResource("lwm2m/device_profile.json", DeviceProfile.class); - } - - deviceProfile.setName(profileName); - deviceProfile.setDefaultQueueName(target.getQueue()); - return tbClient.saveDeviceProfile(deviceProfile); - } - - private void createCalculatedField(TbClient tbClient, Device device) { - log.info("Creating calculated field for device '{}'", device.getName()); - CalculatedField calculatedField = new CalculatedField(); - calculatedField.setName(TEST_CF_TELEMETRY_KEY); - calculatedField.setEntityId(device.getId()); - calculatedField.setType(CalculatedFieldType.SCRIPT); - ScriptCalculatedFieldConfiguration configuration = new ScriptCalculatedFieldConfiguration(); - Argument testDataArgument = new Argument(); - testDataArgument.setRefEntityKey(new ReferencedEntityKey(TEST_TELEMETRY_KEY, ArgumentType.TS_LATEST, null)); - configuration.setArguments(Map.of( - TEST_TELEMETRY_KEY, testDataArgument - )); - configuration.setExpression("return { \"" + TEST_CF_TELEMETRY_KEY + "\": " + TEST_TELEMETRY_KEY + " + \"-cf\" };"); - Output output = new Output(); - output.setType(OutputType.TIME_SERIES); - configuration.setOutput(output); - calculatedField.setConfiguration(configuration); - calculatedField.setDebugMode(true); - tbClient.saveCalculatedField(calculatedField); - } - @Override protected boolean isCfMonitoringEnabled() { return calculatedFieldsMonitoringEnabled; diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/util/ResourceUtils.java b/monitoring/src/main/java/org/thingsboard/monitoring/util/ResourceUtils.java index 3e25cfbd39..a92033f439 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/util/ResourceUtils.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/util/ResourceUtils.java @@ -15,6 +15,7 @@ */ package org.thingsboard.monitoring.util; +import com.fasterxml.jackson.databind.JsonNode; import lombok.SneakyThrows; import org.thingsboard.common.util.JacksonUtil; @@ -24,14 +25,21 @@ public class ResourceUtils { @SneakyThrows public static T getResource(String path, Class type) { - InputStream resource = ResourceUtils.class.getClassLoader().getResourceAsStream(path); - if (resource == null) { - throw new IllegalArgumentException("Resource not found for path " + path); - } + InputStream resource = getResourceStream(path); return JacksonUtil.OBJECT_MAPPER.readValue(resource, type); } + @SneakyThrows + public static JsonNode getResource(String path) { + InputStream resource = getResourceStream(path); + return JacksonUtil.OBJECT_MAPPER.readTree(resource); + } + public static InputStream getResourceAsStream(String path) { + return getResourceStream(path); + } + + private static InputStream getResourceStream(String path) { InputStream resource = ResourceUtils.class.getClassLoader().getResourceAsStream(path); if (resource == null) { throw new IllegalArgumentException("Resource not found for path " + path); diff --git a/monitoring/src/main/resources/root_rule_chain.json b/monitoring/src/main/resources/rule_chain.json similarity index 76% rename from monitoring/src/main/resources/root_rule_chain.json rename to monitoring/src/main/resources/rule_chain.json index a1c12c8e9d..96d2388636 100644 --- a/monitoring/src/main/resources/root_rule_chain.json +++ b/monitoring/src/main/resources/rule_chain.json @@ -1,257 +1,232 @@ { + "version": 1, "ruleChain": { - "additionalInfo": null, "name": "Root Rule Chain", "type": "CORE", "firstRuleNodeId": null, "root": false, "debugMode": false, "configuration": null, - "externalId": null + "additionalInfo": null }, "metadata": { - "firstNodeIndex": 12, + "firstNodeIndex": 9, "nodes": [ { - "additionalInfo": { - "description": null, - "layoutX": 1202, - "layoutY": 221 - }, "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", + "debugSettings": { + "failuresEnabled": true, + "allEnabled": false, + "allEnabledUntil": 1735310701003 + }, "singletonMode": false, + "queueName": null, "configurationVersion": 1, "configuration": { "defaultTTL": 0, - "useServerTs": false, "processingSettings": { "type": "ON_EVERY_MESSAGE" } }, - "externalId": null + "additionalInfo": { + "description": null, + "layoutX": 1202, + "layoutY": 221 + } }, { - "additionalInfo": { - "layoutX": 1000, - "layoutY": 167 - }, "type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode", "name": "Save Attributes", + "debugSettings": null, "singletonMode": false, + "queueName": null, "configurationVersion": 3, "configuration": { + "scope": "CLIENT_SCOPE", + "notifyDevice": false, "processingSettings": { "type": "ON_EVERY_MESSAGE" }, - "scope": "CLIENT_SCOPE", - "notifyDevice": false, "sendAttributesUpdatedNotification": false, "updateAttributesOnlyOnValueChange": false }, - "externalId": null + "additionalInfo": { + "layoutX": 1000, + "layoutY": 167 + } }, { - "additionalInfo": { - "layoutX": 566, - "layoutY": 302 - }, "type": "org.thingsboard.rule.engine.filter.TbMsgTypeSwitchNode", "name": "Message Type Switch", + "debugSettings": null, "singletonMode": false, + "queueName": null, "configurationVersion": 0, "configuration": { "version": 0 }, - "externalId": null + "additionalInfo": { + "layoutX": 566, + "layoutY": 302 + } }, { - "additionalInfo": { - "layoutX": 1000, - "layoutY": 381 - }, "type": "org.thingsboard.rule.engine.action.TbLogNode", "name": "Log RPC from Device", + "debugSettings": null, "singletonMode": false, + "queueName": null, "configurationVersion": 0, "configuration": { "scriptLang": "TBEL", "jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);", "tbelScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);" }, - "externalId": null - }, - { "additionalInfo": { "layoutX": 1000, - "layoutY": 494 - }, + "layoutY": 381 + } + }, + { "type": "org.thingsboard.rule.engine.action.TbLogNode", "name": "Log Other", + "debugSettings": null, "singletonMode": false, + "queueName": null, "configurationVersion": 0, "configuration": { "scriptLang": "TBEL", "jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);", "tbelScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);" }, - "externalId": null - }, - { "additionalInfo": { "layoutX": 1000, - "layoutY": 583 - }, + "layoutY": 494 + } + }, + { "type": "org.thingsboard.rule.engine.rpc.TbSendRPCRequestNode", "name": "RPC Call Request", + "debugSettings": null, "singletonMode": false, + "queueName": null, "configurationVersion": 0, "configuration": { "timeoutInSeconds": 60 }, - "externalId": null - }, - { - "additionalInfo": { - "layoutX": 255, - "layoutY": 301 - }, - "type": "org.thingsboard.rule.engine.filter.TbOriginatorTypeFilterNode", - "name": "Is Entity Group", - "singletonMode": false, - "configurationVersion": 0, - "configuration": { - "originatorTypes": [ - "ENTITY_GROUP" - ] - }, - "externalId": null - }, - { "additionalInfo": { - "layoutX": 319, - "layoutY": 109 - }, - "type": "org.thingsboard.rule.engine.filter.TbMsgTypeFilterNode", - "name": "Post attributes or RPC request", - "singletonMode": false, - "configurationVersion": 0, - "configuration": { - "messageTypes": [ - "POST_ATTRIBUTES_REQUEST", - "RPC_CALL_FROM_SERVER_TO_DEVICE" - ] - }, - "externalId": null + "layoutX": 1000, + "layoutY": 583 + } }, { - "additionalInfo": { - "layoutX": 627, - "layoutY": 108 + "type": "org.thingsboard.rule.engine.profile.TbDeviceProfileNode", + "name": "Device Profile Node", + "debugSettings": { + "failuresEnabled": true, + "allEnabled": false, + "allEnabledUntil": 1735310701003 }, - "type": "org.thingsboard.rule.engine.transform.TbDuplicateMsgToGroupNode", - "name": "Duplicate To Group Entities", "singletonMode": false, - "configurationVersion": 0, + "queueName": null, + "configurationVersion": 1, "configuration": { - "entityGroupId": null, - "entityGroupIsMessageOriginator": true + "persistAlarmRulesState": false, + "fetchAlarmRulesStateOnStart": false }, - "externalId": null - }, - { "additionalInfo": { "description": "Process incoming messages from devices with the alarm rules defined in the device profile. Dispatch all incoming messages with \"Success\" relation type.", "layoutX": 45, "layoutY": 359 - }, - "type": "org.thingsboard.rule.engine.profile.TbDeviceProfileNode", - "name": "Device Profile Node", - "singletonMode": false, - "configurationVersion": 0, - "configuration": { - "persistAlarmRulesState": false, - "fetchAlarmRulesStateOnStart": false - }, - "externalId": null + } }, { - "additionalInfo": { - "description": "", - "layoutX": 160, - "layoutY": 631 - }, "type": "org.thingsboard.rule.engine.filter.TbJsFilterNode", "name": "Test JS script", + "debugSettings": null, "singletonMode": false, + "queueName": null, "configurationVersion": 0, "configuration": { "scriptLang": "JS", - "jsScript": "var test = {\n a: 'a',\n b: 'b'\n};\nreturn test.a === 'a' && test.b === 'b';", + "jsScript": "var test = {\n a: 'a',\n b: 'b'\n};\n\nreturn test.a === 'a' && test.b === 'b';", "tbelScript": "return msg.temperature > 20;" }, - "externalId": null + "additionalInfo": { + "description": "dashboardId: ${MONITORING:dashboardId}", + "layoutX": 251, + "layoutY": 499 + } }, { - "additionalInfo": { - "description": "", - "layoutX": 427, - "layoutY": 541 - }, "type": "org.thingsboard.rule.engine.filter.TbJsFilterNode", "name": "Test TBEL script", + "debugSettings": null, "singletonMode": false, + "queueName": null, "configurationVersion": 0, "configuration": { "scriptLang": "TBEL", "jsScript": "return msg.temperature > 20;", "tbelScript": "var a = \"a\";\nvar b = \"b\";\nreturn a.equals(\"a\") && b.equals(\"b\");" }, - "externalId": null - }, - { "additionalInfo": { "description": "", - "layoutX": 40, - "layoutY": 252 - }, + "layoutX": 317, + "layoutY": 355 + } + }, + { "type": "org.thingsboard.rule.engine.transform.TbTransformMsgNode", "name": "Add arrival timestamp", + "debugSettings": { + "failuresEnabled": true, + "allEnabled": false, + "allEnabledUntil": 1744642101587 + }, "singletonMode": false, + "queueName": null, "configurationVersion": 0, "configuration": { "scriptLang": "TBEL", "jsScript": "return {msg: msg, metadata: metadata, msgType: msgType};", "tbelScript": "metadata.arrivalTs = Date.now();\nreturn {msg: msg, metadata: metadata, msgType: msgType};" }, - "externalId": null - }, - { "additionalInfo": { "description": "", - "layoutX": 1467, - "layoutY": 267 - }, + "layoutX": 40, + "layoutY": 252 + } + }, + { "type": "org.thingsboard.rule.engine.transform.TbTransformMsgNode", "name": "Calculate additional latencies", + "debugSettings": { + "failuresEnabled": true, + "allEnabled": false, + "allEnabledUntil": 1735310701003 + }, "singletonMode": false, + "queueName": null, "configurationVersion": 0, "configuration": { "scriptLang": "TBEL", "jsScript": "return {msg: msg, metadata: metadata, msgType: msgType};", "tbelScript": "var arrivalLatency = metadata.arrivalTs - metadata.ts;\nvar processingTime = Date.now() - metadata.arrivalTs;\nmsg = {\n arrivalLatency: arrivalLatency,\n processingTime: processingTime\n};\nreturn {msg: msg, metadata: metadata, msgType: msgType};" }, - "externalId": null - }, - { "additionalInfo": { "description": "", - "layoutX": 1438, - "layoutY": 403 - }, + "layoutX": 1467, + "layoutY": 267 + } + }, + { "type": "org.thingsboard.rule.engine.transform.TbChangeOriginatorNode", "name": "To latencies asset", + "debugSettings": null, "singletonMode": false, + "queueName": null, "configurationVersion": 0, "configuration": { "originatorSource": "ENTITY", @@ -269,64 +244,79 @@ "fetchLastLevelOnly": false } }, - "externalId": null + "additionalInfo": { + "description": "", + "layoutX": 1438, + "layoutY": 403 + } }, { - "additionalInfo": { - "description": null, - "layoutX": 1458, - "layoutY": 505 - }, "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", + "debugSettings": { + "failuresEnabled": true, + "allEnabled": false, + "allEnabledUntil": 1735310701003 + }, "singletonMode": false, + "queueName": null, "configurationVersion": 1, "configuration": { "defaultTTL": 0, - "useServerTs": false, "processingSettings": { "type": "ON_EVERY_MESSAGE" } }, - "externalId": null + "additionalInfo": { + "description": null, + "layoutX": 1458, + "layoutY": 505 + } }, { - "additionalInfo": { - "description": "", - "layoutX": 928, - "layoutY": 266 - }, "type": "org.thingsboard.rule.engine.filter.TbCheckMessageNode", "name": "Has testData", + "debugSettings": null, "singletonMode": false, + "queueName": null, "configurationVersion": 0, "configuration": { "messageNames": [ - "testData" + "testData", + "testDataCf" ], "metadataNames": [], - "checkAllKeys": true + "checkAllKeys": false }, - "externalId": null + "additionalInfo": { + "description": "", + "layoutX": 928, + "layoutY": 266 + } }, { - "additionalInfo": { - "description": null, - "layoutX": 1203, - "layoutY": 327 - }, "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries with TTL", + "debugSettings": { + "failuresEnabled": true, + "allEnabled": false, + "allEnabledUntil": 1742305233839 + }, "singletonMode": false, + "queueName": null, "configurationVersion": 1, "configuration": { - "defaultTTL": 180, - "useServerTs": false, "processingSettings": { "type": "ON_EVERY_MESSAGE" - } + }, + "defaultTTL": 60, + "useServerTs": null }, - "externalId": null + "additionalInfo": { + "description": "", + "layoutX": 1203, + "layoutY": 327 + } } ], "connections": [ @@ -352,23 +342,13 @@ }, { "fromIndex": 2, - "toIndex": 16, + "toIndex": 13, "type": "Post telemetry" }, - { - "fromIndex": 6, - "toIndex": 2, - "type": "False" - }, { "fromIndex": 6, "toIndex": 7, - "type": "True" - }, - { - "fromIndex": 7, - "toIndex": 2, - "type": "False" + "type": "Success" }, { "fromIndex": 7, @@ -378,54 +358,39 @@ { "fromIndex": 8, "toIndex": 2, - "type": "Success" + "type": "True" }, { "fromIndex": 9, - "toIndex": 10, + "toIndex": 6, "type": "Success" }, { "fromIndex": 10, "toIndex": 11, - "type": "True" + "type": "Success" }, { "fromIndex": 11, - "toIndex": 6, - "type": "True" - }, - { - "fromIndex": 12, - "toIndex": 9, + "toIndex": 12, "type": "Success" }, { "fromIndex": 13, - "toIndex": 14, - "type": "Success" - }, - { - "fromIndex": 14, - "toIndex": 15, - "type": "Success" - }, - { - "fromIndex": 16, "toIndex": 0, "type": "False" }, { - "fromIndex": 16, - "toIndex": 17, + "fromIndex": 13, + "toIndex": 14, "type": "True" }, { - "fromIndex": 17, - "toIndex": 13, + "fromIndex": 14, + "toIndex": 10, "type": "Success" } ], "ruleChainConnections": null } -} +} \ No newline at end of file From 602d60281c5537fba1bfc194135a275b97b66fa4 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 12 May 2025 12:50:03 +0300 Subject: [PATCH 39/40] Fixes for monitoring --- .../config/transport/TransportMonitoringTarget.java | 5 +++++ .../monitoring/service/MonitoringEntityService.java | 9 ++++++++- monitoring/src/main/resources/rule_chain.json | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringTarget.java b/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringTarget.java index e8a9ab03fa..5558f39c05 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringTarget.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringTarget.java @@ -15,6 +15,7 @@ */ package org.thingsboard.monitoring.config.transport; +import com.google.common.base.Strings; import lombok.Data; import org.apache.commons.lang3.StringUtils; import org.thingsboard.monitoring.config.MonitoringTarget; @@ -39,4 +40,8 @@ public class TransportMonitoringTarget implements MonitoringTarget { return StringUtils.defaultIfEmpty(queue, "Main"); } + public String getNamePrefix() { + return Strings.nullToEmpty(namePrefix); + } + } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringEntityService.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringEntityService.java index 66dc9200e3..062104ecd5 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringEntityService.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringEntityService.java @@ -29,6 +29,7 @@ import org.thingsboard.monitoring.config.transport.TransportMonitoringConfig; import org.thingsboard.monitoring.config.transport.TransportMonitoringTarget; import org.thingsboard.monitoring.config.transport.TransportType; import org.thingsboard.monitoring.util.ResourceUtils; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileType; @@ -113,6 +114,8 @@ public class MonitoringEntityService { RuleChainMetaData metaData = JacksonUtil.fromString(metadataJson, RuleChainMetaData.class); metaData.setRuleChainId(ruleChainId); tbClient.saveRuleChainMetaData(metaData); + tbClient.saveEntityAttributesV2(ruleChainId, DataConstants.SERVER_SCOPE, JacksonUtil.newObjectNode() + .put("version", newVersion)); } public Asset getOrCreateMonitoringAsset() { @@ -184,7 +187,11 @@ public class MonitoringEntityService { credentials.setCredentialsValue(JacksonUtil.toString(lwm2mCreds)); } - return tbClient.saveDeviceWithCredentials(device, credentials).get(); + device = tbClient.saveDeviceWithCredentials(device, credentials).get(); + if (calculatedFieldsMonitoringEnabled) { + createCalculatedField(device); + } + return device; } private DeviceProfile getOrCreateDeviceProfile(TransportMonitoringConfig config, TransportMonitoringTarget target) { diff --git a/monitoring/src/main/resources/rule_chain.json b/monitoring/src/main/resources/rule_chain.json index 96d2388636..2bc0e89a8a 100644 --- a/monitoring/src/main/resources/rule_chain.json +++ b/monitoring/src/main/resources/rule_chain.json @@ -154,7 +154,7 @@ "tbelScript": "return msg.temperature > 20;" }, "additionalInfo": { - "description": "dashboardId: ${MONITORING:dashboardId}", + "description": "", "layoutX": 251, "layoutY": 499 } From fd66c5f1773251fde5d016f7351843ea1caef8b3 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 12 May 2025 12:58:33 +0300 Subject: [PATCH 40/40] CF monitoring fixes --- .../thingsboard/monitoring/data/MonitoredServiceKey.java | 1 - .../thingsboard/monitoring/service/BaseHealthChecker.java | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/data/MonitoredServiceKey.java b/monitoring/src/main/java/org/thingsboard/monitoring/data/MonitoredServiceKey.java index 7579d75231..9c3ee5b786 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/data/MonitoredServiceKey.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/data/MonitoredServiceKey.java @@ -19,6 +19,5 @@ public class MonitoredServiceKey { public static final String GENERAL = "Monitoring"; public static final String EDQS = "*EDQS*"; - public static final String CF = "*CF*"; } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java index d19e723332..072310fb67 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java @@ -117,11 +117,9 @@ public abstract class BaseHealthChecker