Browse Source

Merge branch 'refs/heads/master' into lwm2m_observe_composite

pull/13279/head
nickAS21 1 year ago
parent
commit
ccad309cfc
  1. 5
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  2. 14
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
  3. 2
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java
  4. 2
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java
  5. 37
      application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientRetransmissionSettingsComponent.java
  6. 47
      application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientSettingsComponent.java
  7. 2
      application/src/main/java/org/thingsboard/server/controller/AlarmController.java
  8. 5
      application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java
  9. 80
      application/src/main/java/org/thingsboard/server/controller/TrendzController.java
  10. 3
      application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java
  11. 14
      application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java
  12. 23
      application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java
  13. 3
      application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java
  14. 2
      application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java
  15. 46
      application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java
  16. 17
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  17. 3
      application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java
  18. 8
      application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java
  19. 9
      application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java
  20. 2
      application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java
  21. 2
      application/src/main/java/org/thingsboard/server/service/ruleengine/DefaultRuleEngineCallService.java
  22. 2
      application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java
  23. 22
      application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java
  24. 9
      application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java
  25. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java
  26. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java
  27. 45
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java
  28. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java
  29. 6
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java
  30. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java
  31. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java
  32. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java
  33. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationRuleImportService.java
  34. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationTargetImportService.java
  35. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationTemplateImportService.java
  36. 20
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java
  37. 14
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java
  38. 6
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetTypeImportService.java
  39. 6
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java
  40. 2
      application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
  41. 2
      application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java
  42. 33
      application/src/main/resources/thingsboard.yml
  43. 12
      application/src/test/java/org/thingsboard/server/actors/rule/DefaultTbContextTest.java
  44. 30
      application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java
  45. 2
      application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java
  46. 4
      application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java
  47. 3
      application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java
  48. 77
      application/src/test/java/org/thingsboard/server/controller/TrendzControllerTest.java
  49. 163
      application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java
  50. 2
      application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java
  51. 2
      application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineStrategyTest.java
  52. 2
      application/src/test/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcServiceTest.java
  53. 2
      application/src/test/java/org/thingsboard/server/service/ruleengine/DefaultRuleEngineCallServiceTest.java
  54. 43
      application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java
  55. 29
      common/dao-api/src/main/java/org/thingsboard/server/dao/trendz/TrendzSettingsService.java
  56. 1
      common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java
  57. 2
      common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java
  58. 8
      common/data/src/main/java/org/thingsboard/server/common/data/TbProperty.java
  59. 1
      common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java
  60. 6
      common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java
  61. 26
      common/data/src/main/java/org/thingsboard/server/common/data/trendz/TrendzSettings.java
  62. 104
      common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java
  63. 4
      common/message/src/main/proto/tbmsg.proto
  64. 22
      common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java
  65. 8
      common/proto/src/main/proto/queue.proto
  66. 2
      common/queue/src/main/java/org/thingsboard/server/queue/common/TbRuleEngineProducerService.java
  67. 38
      common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java
  68. 17
      common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java
  69. 20
      common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java
  70. 37
      dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java
  71. 6
      dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java
  72. 69
      dao/src/main/java/org/thingsboard/server/dao/trendz/DefaultTrendzSettingsService.java
  73. 125
      dao/src/test/java/org/thingsboard/server/dao/service/RuleChainServiceTest.java
  74. 3
      dao/src/test/resources/application-test.properties
  75. 6
      edqs/src/main/resources/edqs.yml
  76. 12
      msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java
  77. 15
      msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java
  78. 14
      msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java
  79. 5
      msa/vc-executor/src/main/resources/tb-vc-executor.yml
  80. 20
      netty-mqtt/pom.xml
  81. 24
      netty-mqtt/src/main/java/org/thingsboard/mqtt/MaxRetransmissionsReachedException.java
  82. 30
      netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttChannelHandler.java
  83. 2
      netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClient.java
  84. 20
      netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientConfig.java
  85. 82
      netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java
  86. 3
      netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttConnectResult.java
  87. 134
      netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingPublish.java
  88. 119
      netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingSubscription.java
  89. 85
      netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingUnsubscription.java
  90. 17
      netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPingHandler.java
  91. 4
      netty-mqtt/src/main/java/org/thingsboard/mqtt/PendingOperation.java
  92. 101
      netty-mqtt/src/main/java/org/thingsboard/mqtt/RetransmissionHandler.java
  93. 210
      netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttClientTest.java
  94. 63
      netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttPingHandlerTest.java
  95. 202
      netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttTestProxy.java
  96. 151
      netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/MqttIntegrationTest.java
  97. 84
      netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttServer.java
  98. 141
      netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttTransportHandler.java
  99. 3
      netty-mqtt/src/test/resources/junit-platform.properties
  100. 6
      pom.xml

5
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -32,6 +32,7 @@ import org.springframework.stereotype.Component;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.rule.engine.api.DeviceStateManager;
import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.rule.engine.api.MqttClientSettings;
import org.thingsboard.rule.engine.api.NotificationCenter;
import org.thingsboard.rule.engine.api.SmsService;
import org.thingsboard.rule.engine.api.notification.SlackService;
@ -639,6 +640,10 @@ public class ActorSystemContext {
@Getter
private long cfCalculationResultTimeout;
@Autowired
@Getter
private MqttClientSettings mqttClientSettings;
@Getter
@Setter
private TbActorSystem actorSystem;

14
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java

@ -23,14 +23,15 @@ import org.bouncycastle.util.Arrays;
import org.thingsboard.common.util.DebugModeUtil;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ListeningExecutor;
import org.thingsboard.rule.engine.api.DeviceStateManager;
import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.rule.engine.api.MqttClientSettings;
import org.thingsboard.rule.engine.api.NotificationCenter;
import org.thingsboard.rule.engine.api.RuleEngineAlarmService;
import org.thingsboard.rule.engine.api.RuleEngineApiUsageStateService;
import org.thingsboard.rule.engine.api.RuleEngineAssetProfileCache;
import org.thingsboard.rule.engine.api.RuleEngineCalculatedFieldQueueService;
import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache;
import org.thingsboard.rule.engine.api.DeviceStateManager;
import org.thingsboard.rule.engine.api.RuleEngineRpcService;
import org.thingsboard.rule.engine.api.RuleEngineTelemetryService;
import org.thingsboard.rule.engine.api.ScriptEngine;
@ -234,7 +235,8 @@ public class DefaultTbContext implements TbContext {
TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(getTenantId().getId().getMostSignificantBits())
.setTenantIdLSB(getTenantId().getId().getLeastSignificantBits())
.setTbMsg(TbMsg.toByteString(tbMsg)).build();
.setTbMsgProto(TbMsg.toProto(tbMsg))
.build();
mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, callback);
}
@ -313,7 +315,7 @@ public class DefaultTbContext implements TbContext {
TransportProtos.ToRuleEngineMsg.Builder msg = TransportProtos.ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(getTenantId().getId().getMostSignificantBits())
.setTenantIdLSB(getTenantId().getId().getLeastSignificantBits())
.setTbMsg(TbMsg.toByteString(tbMsg))
.setTbMsgProto(TbMsg.toProto(tbMsg))
.addAllRelationTypes(relationTypes);
if (failureMessage != null) {
msg.setFailureMessage(failureMessage);
@ -1010,13 +1012,17 @@ public class DefaultTbContext implements TbContext {
return mainCtx.getAuditLogService();
}
@Override
public MqttClientSettings getMqttClientSettings() {
return mainCtx.getMqttClientSettings();
}
private TbMsgMetaData getActionMetaData(RuleNodeId ruleNodeId) {
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("ruleNodeId", ruleNodeId.toString());
return metaData;
}
@Override
public void schedule(Runnable runnable, long delay, TimeUnit timeUnit) {
mainCtx.getScheduler().schedule(runnable, delay, timeUnit);

2
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java

@ -381,7 +381,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
ToRuleEngineMsg toQueueMsg = ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setTbMsg(TbMsg.toByteString(newMsg))
.setTbMsgProto(TbMsg.toProto(newMsg))
.build();
clusterService.pushMsgToRuleEngine(tpi, newMsg.getId(), toQueueMsg, callbackWrapper);
}

2
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java

@ -210,7 +210,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
TransportProtos.ToRuleEngineMsg toQueueMsg = TransportProtos.ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setTbMsg(TbMsg.toByteString(tbMsg))
.setTbMsgProto(TbMsg.toProto(tbMsg))
.build();
systemContext.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), toQueueMsg, null);
defaultCtx.ack(source);

37
application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientRetransmissionSettingsComponent.java

@ -0,0 +1,37 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.config.mqtt;
import jakarta.validation.constraints.PositiveOrZero;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.validation.annotation.Validated;
@Data
@Validated
@Configuration
@ConfigurationProperties(prefix = "mqtt.client.retransmission")
public class MqttClientRetransmissionSettingsComponent {
@PositiveOrZero
private int maxAttempts;
@PositiveOrZero
private long initialDelayMillis;
@PositiveOrZero
private double jitterFactor;
}

47
application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientSettingsComponent.java

@ -0,0 +1,47 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.config.mqtt;
import lombok.EqualsAndHashCode;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
import org.springframework.context.annotation.Configuration;
import org.thingsboard.rule.engine.api.MqttClientSettings;
@ToString
@EqualsAndHashCode
@Configuration
@RequiredArgsConstructor
public class MqttClientSettingsComponent implements MqttClientSettings {
private final MqttClientRetransmissionSettingsComponent retransmissionSettingsComponent;
@Override
public int getRetransmissionMaxAttempts() {
return retransmissionSettingsComponent.getMaxAttempts();
}
@Override
public long getRetransmissionInitialDelayMillis() {
return retransmissionSettingsComponent.getInitialDelayMillis();
}
@Override
public double getRetransmissionJitterFactor() {
return retransmissionSettingsComponent.getJitterFactor();
}
}

2
application/src/main/java/org/thingsboard/server/controller/AlarmController.java

@ -157,7 +157,7 @@ public class AlarmController extends BaseController {
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
@RequestMapping(value = "/alarm/{alarmId}", method = RequestMethod.DELETE)
@ResponseBody
public Boolean deleteAlarm(@Parameter(description = ALARM_ID_PARAM_DESCRIPTION) @PathVariable(ALARM_ID) String strAlarmId) throws ThingsboardException {
public boolean deleteAlarm(@Parameter(description = ALARM_ID_PARAM_DESCRIPTION) @PathVariable(ALARM_ID) String strAlarmId) throws ThingsboardException {
checkParameter(ALARM_ID, strAlarmId);
AlarmId alarmId = new AlarmId(toUUID(strAlarmId));
Alarm alarm = checkAlarmId(alarmId, Operation.DELETE);

5
application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java

@ -42,6 +42,7 @@ import org.thingsboard.server.common.data.settings.UserSettings;
import org.thingsboard.server.common.data.settings.UserSettingsType;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.dao.mobile.QrCodeSettingService;
import org.thingsboard.server.dao.trendz.TrendzSettingsService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.model.UserPrincipal;
@ -87,6 +88,9 @@ public class SystemInfoController extends BaseController {
@Autowired
private DebugModeRateLimitsConfig debugModeRateLimitsConfig;
@Autowired
private TrendzSettingsService trendzSettingsService;
@PostConstruct
public void init() {
JsonNode info = buildInfoObject();
@ -158,6 +162,7 @@ public class SystemInfoController extends BaseController {
}
systemParams.setMaxArgumentsPerCF(tenantProfileConfiguration.getMaxArgumentsPerCF());
systemParams.setMaxDataPointsPerRollingArg(tenantProfileConfiguration.getMaxDataPointsPerRollingArg());
systemParams.setTrendzSettings(trendzSettingsService.findTrendzSettings(currentUser.getTenantId()));
}
systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID))
.map(QrCodeSettings::getQrCodeConfig).map(QRCodeConfig::isShowOnHomePage)

80
application/src/main/java/org/thingsboard/server/controller/TrendzController.java

@ -0,0 +1,80 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.controller;
import lombok.RequiredArgsConstructor;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.core.annotation.AuthenticationPrincipal;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.trendz.TrendzSettings;
import org.thingsboard.server.config.annotations.ApiOperation;
import org.thingsboard.server.dao.trendz.TrendzSettingsService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;
import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END;
import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START;
import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE;
import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH;
import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH;
@RestController
@TbCoreComponent
@RequiredArgsConstructor
@RequestMapping("/api")
public class TrendzController extends BaseController {
private final TrendzSettingsService trendzSettingsService;
@ApiOperation(value = "Save Trendz settings (saveTrendzSettings)",
notes = "Saves Trendz settings for this tenant.\n" + NEW_LINE +
"Here is an example of the Trendz settings:\n" +
MARKDOWN_CODE_BLOCK_START +
"{\n" +
" \"enabled\": true,\n" +
" \"baseUrl\": \"https://some.domain.com:18888/also_necessary_prefix\"\n" +
"}" +
MARKDOWN_CODE_BLOCK_END +
TENANT_AUTHORITY_PARAGRAPH)
@PostMapping("/trendz/settings")
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
public TrendzSettings saveTrendzSettings(@RequestBody TrendzSettings trendzSettings,
@AuthenticationPrincipal SecurityUser user) throws ThingsboardException {
accessControlService.checkPermission(user, Resource.ADMIN_SETTINGS, Operation.WRITE);
TenantId tenantId = user.getTenantId();
trendzSettingsService.saveTrendzSettings(tenantId, trendzSettings);
return trendzSettings;
}
@ApiOperation(value = "Get Trendz Settings (getTrendzSettings)",
notes = "Retrieves Trendz settings for this tenant." +
TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH)
@GetMapping("/trendz/settings")
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
public TrendzSettings getTrendzSettings(@AuthenticationPrincipal SecurityUser user) {
TenantId tenantId = user.getTenantId();
return trendzSettingsService.findTrendzSettings(tenantId);
}
}

3
application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java

@ -284,7 +284,8 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService {
protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) {
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator());
TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg))
TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder()
.setTbMsgProto(TbMsg.toProto(tbMsg))
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build();
ruleEngineMsgProducer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback);

14
application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java

@ -17,10 +17,8 @@ package org.thingsboard.server.service.entitiy;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.core.env.Environment;
import org.thingsboard.server.cluster.TbClusterService;
@ -31,16 +29,10 @@ import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.dao.alarm.AlarmService;
import org.thingsboard.server.dao.asset.AssetProfileService;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import org.thingsboard.server.service.telemetry.AlarmSubscriptionService;
@ -55,12 +47,6 @@ public abstract class AbstractTbEntityService {
@Autowired
private Environment env;
@Value("${server.log_controller_error_stack_trace}")
@Getter
private boolean logControllerErrorStackTrace;
@Autowired
protected DbCallbackExecutorService dbExecutor;
@Autowired(required = false)
protected TbLogEntityActionService logEntityActionService;
@Autowired(required = false)

23
application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java

@ -190,11 +190,24 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb
}
@Override
public Boolean delete(Alarm alarm, User user) {
TenantId tenantId = alarm.getTenantId();
logEntityActionService.logEntityAction(tenantId, alarm.getOriginator(), alarm, alarm.getCustomerId(),
ActionType.ALARM_DELETE, user, alarm.getId());
return alarmSubscriptionService.deleteAlarm(tenantId, alarm.getId());
public boolean delete(Alarm alarm, User user) {
var tenantId = alarm.getTenantId();
var alarmId = alarm.getId();
var alarmOriginator = alarm.getOriginator();
boolean deleted;
try {
deleted = alarmSubscriptionService.deleteAlarm(tenantId, alarmId);
} catch (Exception e) {
logEntityActionService.logEntityAction(tenantId, emptyId(alarmOriginator.getEntityType()), ActionType.ALARM_DELETE, user, e, alarmId);
throw e;
}
if (deleted) {
logEntityActionService.logEntityAction(tenantId, alarmOriginator, alarm, alarm.getCustomerId(), ActionType.ALARM_DELETE, user, alarmId);
}
return deleted;
}
private static long getOrDefault(long ts) {

3
application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java

@ -43,5 +43,6 @@ public interface TbAlarmService {
void unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, List<UUID> alarms, long unassignTs);
Boolean delete(Alarm alarm, User user);
boolean delete(Alarm alarm, User user);
}

2
application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java

@ -86,7 +86,7 @@ public class DefaultTbOtaPackageService extends AbstractTbEntityService implemen
otaPackage.setContentType(contentType);
otaPackage.setData(ByteBuffer.wrap(data));
otaPackage.setDataSize((long) data.length);
OtaPackageInfo savedOtaPackage = otaPackageService.saveOtaPackage(otaPackage);
OtaPackageInfo savedOtaPackage = new OtaPackageInfo(otaPackageService.saveOtaPackage(otaPackage));
logEntityActionService.logEntityAction(tenantId, savedOtaPackage.getId(), savedOtaPackage, null, actionType, user);
return savedOtaPackage;
} catch (Exception e) {

46
application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java

@ -25,21 +25,30 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.alarm.AlarmSeverity;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageDataIterable;
import org.thingsboard.server.common.data.query.DynamicValue;
import org.thingsboard.server.common.data.query.FilterPredicateValue;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.sql.JpaExecutorService;
import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.service.component.RuleNodeClassInfo;
import org.thingsboard.server.service.install.DbUpgradeExecutorService;
import org.thingsboard.server.utils.TbNodeUpgradeUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import static org.thingsboard.server.dao.rule.BaseRuleChainService.TB_RULE_CHAIN_INPUT_NODE;
@Service
@Profile("install")
@Slf4j
@ -51,19 +60,50 @@ public class DefaultDataUpdateService implements DataUpdateService {
@Autowired
private RuleChainService ruleChainService;
@Autowired
private RelationService relationService;
@Autowired
private ComponentDiscoveryService componentDiscoveryService;
@Autowired
JpaExecutorService jpaExecutorService;
private DbUpgradeExecutorService executorService;
@Override
public void updateData() throws Exception {
log.info("Updating data ...");
//TODO: should be cleaned after each release
updateInputNodes();
log.info("Data updated.");
}
private void updateInputNodes() {
log.info("Creating relations for input nodes...");
int n = 0;
var inputNodes = new PageDataIterable<>(pageLink -> ruleChainService.findAllRuleNodesByType(TB_RULE_CHAIN_INPUT_NODE, pageLink), 1024);
for (RuleNode inputNode : inputNodes) {
try {
RuleChainId targetRuleChainId = Optional.ofNullable(inputNode.getConfiguration().get("ruleChainId"))
.filter(JsonNode::isTextual).map(JsonNode::asText).map(id -> new RuleChainId(UUID.fromString(id)))
.orElse(null);
if (targetRuleChainId == null) {
continue;
}
EntityRelation relation = new EntityRelation();
relation.setFrom(inputNode.getRuleChainId());
relation.setTo(targetRuleChainId);
relation.setType(EntityRelation.USES_TYPE);
relation.setTypeGroup(RelationTypeGroup.COMMON);
relationService.saveRelation(TenantId.SYS_TENANT_ID, relation);
n++;
} catch (Exception e) {
log.error("Failed to save relation for input node: {}", inputNode, e);
}
}
log.info("Created {} relations for input nodes", n);
}
@Override
public void upgradeRuleNodes() {
int totalRuleNodesUpgraded = 0;
@ -107,7 +147,7 @@ public class DefaultDataUpdateService implements DataUpdateService {
ruleNodeId, ruleNodeType, fromVersion, toVersion);
try {
TbNodeUpgradeUtils.upgradeConfigurationAndVersion(ruleNode, ruleNodeClassInfo);
saveFutures.add(jpaExecutorService.submit(() -> {
saveFutures.add(executorService.submit(() -> {
ruleChainService.saveRuleNode(TenantId.SYS_TENANT_ID, ruleNode);
log.debug("Successfully upgrade rule node with id: {} type: {} fromVersion: {} toVersion: {}",
ruleNodeId, ruleNodeType, fromVersion, toVersion);

17
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -552,19 +552,10 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
proto.getScope(), KvProtoUtil.toAttributeKvList(proto.getDataList()), callback);
} else if (msg.hasAttrDelete()) {
TbAttributeDeleteProto proto = msg.getAttrDelete();
if (proto.hasNotifyDevice()) {
// handles old messages with deprecated 'notifyDevice'
subscriptionManagerService.onAttributesDelete(
toTenantId(proto.getTenantIdMSB(), proto.getTenantIdLSB()),
TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()),
proto.getScope(), proto.getKeysList(), proto.getNotifyDevice(), callback);
} else {
// handles new messages without 'notifyDevice'
subscriptionManagerService.onAttributesDelete(
toTenantId(proto.getTenantIdMSB(), proto.getTenantIdLSB()),
TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()),
proto.getScope(), proto.getKeysList(), callback);
}
subscriptionManagerService.onAttributesDelete(
toTenantId(proto.getTenantIdMSB(), proto.getTenantIdLSB()),
TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()),
proto.getScope(), proto.getKeysList(), callback);
} else if (msg.hasTsDelete()) {
TbTimeSeriesDeleteProto proto = msg.getTsDelete();
subscriptionManagerService.onTimeSeriesDelete(

3
application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java

@ -20,6 +20,7 @@ import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.msg.gen.MsgProtos;
import org.thingsboard.server.common.util.ProtoUtils;
import org.thingsboard.server.gen.transport.TransportProtos;
import java.util.UUID;
@ -34,7 +35,7 @@ public class SequentialByOriginatorIdTbRuleEngineSubmitStrategy extends Sequenti
@Override
protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) {
try {
MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(msg.getTbMsg());
MsgProtos.TbMsgProto proto = ProtoUtils.getTbMsgProto(msg);
return EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()));
} catch (InvalidProtocolBufferException e) {
log.warn("[{}] Failed to parse TbMsg: {}", queueName, msg);

8
application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java

@ -19,8 +19,8 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import org.thingsboard.server.common.data.queue.ProcessingStrategy;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import org.thingsboard.server.common.util.ProtoUtils;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -125,7 +125,7 @@ public class TbRuleEngineProcessingStrategyFactory {
}
log.debug("[{}] Going to reprocess {} messages", queueName, toReprocess.size());
if (log.isTraceEnabled()) {
toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(result.getQueueName(), msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, ProtoUtils.fromTbMsgProto(result.getQueueName(), msg.getValue(), TbMsgCallback.EMPTY)));
}
if (pauseBetweenRetries > 0) {
try {
@ -164,10 +164,10 @@ public class TbRuleEngineProcessingStrategyFactory {
log.debug("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size());
}
if (log.isTraceEnabled()) {
result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(result.getQueueName(), msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, ProtoUtils.fromTbMsgProto(result.getQueueName(), msg.getValue(), TbMsgCallback.EMPTY)));
}
if (log.isTraceEnabled()) {
result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, TbMsg.fromBytes(result.getQueueName(), msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, ProtoUtils.fromTbMsgProto(result.getQueueName(), msg.getValue(), TbMsgCallback.EMPTY)));
}
return new TbRuleEngineProcessingDecision(true, null);
}

9
application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java

@ -30,6 +30,7 @@ import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.common.util.ProtoUtils;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -178,7 +179,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager<T
new TbMsgPackCallback(id, tenantId, packCtx, stats.getTimer(tenantId, SUCCESSFUL_STATUS), stats.getTimer(tenantId, FAILED_STATUS)) :
new TbMsgPackCallback(id, tenantId, packCtx);
try {
if (!toRuleEngineMsg.getTbMsg().isEmpty()) {
if (!toRuleEngineMsg.getTbMsg().isEmpty() || toRuleEngineMsg.hasTbMsgProto()) {
forwardToRuleEngineActor(config.getName(), tenantId, toRuleEngineMsg, callback);
} else {
callback.onSuccess();
@ -189,7 +190,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager<T
}
private void forwardToRuleEngineActor(String queueName, TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) {
TbMsg tbMsg = TbMsg.fromBytes(queueName, toRuleEngineMsg.getTbMsg().toByteArray(), callback);
TbMsg tbMsg = ProtoUtils.fromTbMsgProto(queueName, toRuleEngineMsg, callback);
QueueToRuleEngineMsg msg;
ProtocolStringList relationTypesList = toRuleEngineMsg.getRelationTypesList();
Set<String> relationTypes;
@ -207,7 +208,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager<T
log.info("[{}] {} to process [{}] messages", queueKey, prefix, map.size());
for (Map.Entry<UUID, TbProtoQueueMsg<ToRuleEngineMsg>> pending : map.entrySet()) {
ToRuleEngineMsg tmp = pending.getValue().getValue();
TbMsg tmpMsg = TbMsg.fromBytes(config.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY);
TbMsg tmpMsg = ProtoUtils.fromTbMsgProto(config.getName(), tmp, TbMsgCallback.EMPTY);
RuleNodeInfo ruleNodeInfo = ctx.getLastVisitedRuleNode(pending.getKey());
if (printAll) {
log.trace("[{}][{}] {} to process message: {}, Last Rule Node: {}", queueKey, TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo);
@ -236,7 +237,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager<T
}
for (TbProtoQueueMsg<ToRuleEngineMsg> msg : msgs) {
try {
MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray());
MsgProtos.TbMsgProto tbMsgProto = ProtoUtils.getTbMsgProto(msg.getValue());
EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB()));
TopicPartitionInfo tpi = ctx.getPartitionService().resolve(ServiceType.TB_RULE_ENGINE, config.getName(), TenantId.SYS_TENANT_ID, originator);

2
application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java

@ -133,7 +133,7 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi
TransportProtos.RestApiCallResponseMsgProto msg = TransportProtos.RestApiCallResponseMsgProto.newBuilder()
.setRequestIdMSB(requestId.getMostSignificantBits())
.setRequestIdLSB(requestId.getLeastSignificantBits())
.setResponse(TbMsg.toByteString(tbMsg))
.setResponseProto(TbMsg.toProto(tbMsg))
.build();
clusterService.pushNotificationToCore(serviceId, msg, null);
}

2
application/src/main/java/org/thingsboard/server/service/ruleengine/DefaultRuleEngineCallService.java

@ -73,7 +73,7 @@ public class DefaultRuleEngineCallService implements RuleEngineCallService {
UUID requestId = new UUID(restApiCallResponseMsg.getRequestIdMSB(), restApiCallResponseMsg.getRequestIdLSB());
Consumer<TbMsg> consumer = requests.remove(requestId);
if (consumer != null) {
consumer.accept(TbMsg.fromBytes(null, restApiCallResponseMsg.getResponse().toByteArray(), TbMsgCallback.EMPTY));
consumer.accept(TbMsg.fromProto(null, restApiCallResponseMsg.getResponseProto(), restApiCallResponseMsg.getResponse(), TbMsgCallback.EMPTY));
} else {
log.trace("[{}] Unknown or stale rest api call response received", requestId);
}

2
application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java

@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.service.security.model.SecurityUser;
@Component(value="sysAdminPermissions")
@Component(value = "sysAdminPermissions")
public class SysAdminPermissions extends AbstractPermissions {
public SysAdminPermissions() {

22
application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java

@ -20,10 +20,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.alarm.AlarmInfo;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId;
@ -36,7 +33,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.common.msg.rule.engine.DeviceAttributesEventNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -73,7 +69,6 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene
private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueProducerProvider producerProvider;
private final TbLocalSubscriptionService localSubscriptionService;
private final TbClusterService clusterService;
private final SubscriptionSchedulerComponent scheduler;
private final Lock subsLock = new ReentrantLock();
@ -204,15 +199,14 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene
@Override
public void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List<String> keys, TbCallback callback) {
onAttributesDelete(tenantId, entityId, scope, keys, false, callback);
}
@Override
public void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List<String> keys, boolean notifyDevice, TbCallback callback) {
processAttributesUpdate(entityId, scope,
keys.stream().map(key -> new BaseAttributeKvEntry(0, new StringDataEntry(key, ""))).collect(Collectors.toList()));
if (entityId.getEntityType() == EntityType.DEVICE && TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope) && notifyDevice) {
clusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onDelete(tenantId, new DeviceId(entityId.getId()), scope, keys), null);
try {
List<AttributeKvEntry> deletedEntries = keys.stream()
.<AttributeKvEntry>map(key -> new BaseAttributeKvEntry(0L, new StringDataEntry(key, "")))
.toList();
processAttributesUpdate(entityId, scope, deletedEntries);
} catch (Exception e) {
callback.onFailure(e);
return;
}
callback.onSuccess();
}

9
application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java

@ -41,15 +41,6 @@ public interface SubscriptionManagerService extends ApplicationListener<Partitio
void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List<String> keys, TbCallback empty);
/**
* This method is retained solely for backwards compatibility, specifically to handle
* legacy proto messages that include the notifyDevice field.
*
* @deprecated as of 4.0, this method will be removed in future releases.
*/
@Deprecated(forRemoval = true, since = "4.0")
void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List<String> keys, boolean notifyDevice, TbCallback empty);
void onTimeSeriesDelete(TenantId tenantId, EntityId entityId, List<String> keys, TbCallback callback);
void onAlarmUpdate(TenantId tenantId, EntityId entityId, AlarmInfo alarm, TbCallback callback);

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java

@ -46,7 +46,7 @@ public class AssetImportService extends BaseEntityImportService<AssetId, Asset,
}
@Override
protected Asset saveOrUpdate(EntitiesImportCtx ctx, Asset asset, EntityExportData<Asset> exportData, IdProvider idProvider) {
protected Asset saveOrUpdate(EntitiesImportCtx ctx, Asset asset, EntityExportData<Asset> exportData, IdProvider idProvider, CompareResult compareResult) {
Asset savedAsset = assetService.saveAsset(asset);
if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) {
importCalculatedFields(ctx, savedAsset, exportData, idProvider);

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java

@ -49,7 +49,7 @@ public class AssetProfileImportService extends BaseEntityImportService<AssetProf
}
@Override
protected AssetProfile saveOrUpdate(EntitiesImportCtx ctx, AssetProfile assetProfile, EntityExportData<AssetProfile> exportData, IdProvider idProvider) {
protected AssetProfile saveOrUpdate(EntitiesImportCtx ctx, AssetProfile assetProfile, EntityExportData<AssetProfile> exportData, IdProvider idProvider, CompareResult compareResult) {
AssetProfile saved = assetProfileService.saveAssetProfile(assetProfile);
if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) {
importCalculatedFields(ctx, saved, exportData, idProvider);

45
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java

@ -18,6 +18,8 @@ package org.thingsboard.server.service.sync.ie.importing.impl;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.api.client.util.Objects;
import com.google.common.util.concurrent.FutureCallback;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.checkerframework.checker.nullness.qual.Nullable;
@ -117,10 +119,10 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
E prepared = prepare(ctx, entity, existingEntity, exportData, idProvider);
boolean saveOrUpdate = existingEntity == null || compare(ctx, exportData, prepared, existingEntity);
CompareResult compareResult = compare(ctx, exportData, prepared, existingEntity);
if (saveOrUpdate) {
E savedEntity = saveOrUpdate(ctx, prepared, exportData, idProvider);
if (compareResult.isUpdateNeeded()) {
E savedEntity = saveOrUpdate(ctx, prepared, exportData, idProvider, compareResult);
boolean created = existingEntity == null;
importResult.setCreated(created);
importResult.setUpdated(!created);
@ -137,6 +139,17 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
return importResult;
}
@Data
@AllArgsConstructor
static class CompareResult {
private boolean updateNeeded;
private boolean externalIdChangedOnly;
public CompareResult(boolean updateNeeded) {
this.updateNeeded = updateNeeded;
}
}
protected boolean updateRelatedEntitiesIfUnmodified(EntitiesImportCtx ctx, E prepared, D exportData, IdProvider idProvider) {
return importCalculatedFields(ctx, prepared, exportData, idProvider);
}
@ -148,18 +161,30 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
protected abstract E prepare(EntitiesImportCtx ctx, E entity, E oldEntity, D exportData, IdProvider idProvider);
protected boolean compare(EntitiesImportCtx ctx, D exportData, E prepared, E existing) {
protected CompareResult compare(EntitiesImportCtx ctx, D exportData, E prepared, E existing) {
if (existing == null) {
log.debug("[{}] Found new entity.", prepared.getId());
return new CompareResult(true);
}
var newCopy = deepCopy(prepared);
var existingCopy = deepCopy(existing);
cleanupForComparison(newCopy);
cleanupForComparison(existingCopy);
var result = !newCopy.equals(existingCopy);
if (result) {
var updateNeeded = isUpdateNeeded(ctx, exportData, newCopy, existingCopy);
boolean externalIdChangedOnly = false;
if (updateNeeded) {
log.debug("[{}] Found update.", prepared.getId());
log.debug("[{}] From: {}", prepared.getId(), newCopy);
log.debug("[{}] To: {}", prepared.getId(), existingCopy);
cleanupExternalId(newCopy);
cleanupExternalId(existingCopy);
externalIdChangedOnly = newCopy.equals(existingCopy);
}
return result;
return new CompareResult(updateNeeded, externalIdChangedOnly);
}
protected boolean isUpdateNeeded(EntitiesImportCtx ctx, D exportData, E prepared, E existing) {
return !prepared.equals(existing);
}
protected abstract E deepCopy(E e);
@ -172,7 +197,11 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
}
}
protected abstract E saveOrUpdate(EntitiesImportCtx ctx, E entity, D exportData, IdProvider idProvider);
protected void cleanupExternalId(E e) {
e.setExternalId(null);
}
protected abstract E saveOrUpdate(EntitiesImportCtx ctx, E entity, D exportData, IdProvider idProvider, CompareResult compareResult);
protected void processAfterSaved(EntitiesImportCtx ctx, EntityImportResult<E> importResult, D exportData, IdProvider idProvider) throws ThingsboardException {

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java

@ -52,7 +52,7 @@ public class CustomerImportService extends BaseEntityImportService<CustomerId, C
}
@Override
protected Customer saveOrUpdate(EntitiesImportCtx ctx, Customer customer, EntityExportData<Customer> exportData, IdProvider idProvider) {
protected Customer saveOrUpdate(EntitiesImportCtx ctx, Customer customer, EntityExportData<Customer> exportData, IdProvider idProvider, CompareResult compareResult) {
if (!customer.isPublic()) {
return customerService.saveCustomer(customer);
} else {

6
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java

@ -75,7 +75,7 @@ public class DashboardImportService extends BaseEntityImportService<DashboardId,
}
@Override
protected Dashboard saveOrUpdate(EntitiesImportCtx ctx, Dashboard dashboard, EntityExportData<Dashboard> exportData, IdProvider idProvider) {
protected Dashboard saveOrUpdate(EntitiesImportCtx ctx, Dashboard dashboard, EntityExportData<Dashboard> exportData, IdProvider idProvider, CompareResult compareResult) {
var tenantId = ctx.getTenantId();
Set<ShortCustomerInfo> assignedCustomers = Optional.ofNullable(dashboard.getAssignedCustomers()).orElse(Collections.emptySet()).stream()
@ -116,8 +116,8 @@ public class DashboardImportService extends BaseEntityImportService<DashboardId,
}
@Override
protected boolean compare(EntitiesImportCtx ctx, EntityExportData<Dashboard> exportData, Dashboard prepared, Dashboard existing) {
return super.compare(ctx, exportData, prepared, existing) || !prepared.getConfiguration().equals(existing.getConfiguration());
protected boolean isUpdateNeeded(EntitiesImportCtx ctx, EntityExportData<Dashboard> exportData, Dashboard prepared, Dashboard existing) {
return super.isUpdateNeeded(ctx, exportData, prepared, existing) || !prepared.getConfiguration().equals(existing.getConfiguration());
}
@Override

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java

@ -63,7 +63,7 @@ public class DeviceImportService extends BaseEntityImportService<DeviceId, Devic
}
@Override
protected Device saveOrUpdate(EntitiesImportCtx ctx, Device device, DeviceExportData exportData, IdProvider idProvider) {
protected Device saveOrUpdate(EntitiesImportCtx ctx, Device device, DeviceExportData exportData, IdProvider idProvider, CompareResult compareResult) {
Device savedDevice;
if (exportData.getCredentials() != null && ctx.isSaveCredentials()) {
exportData.getCredentials().setId(null);

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java

@ -51,7 +51,7 @@ public class DeviceProfileImportService extends BaseEntityImportService<DevicePr
}
@Override
protected DeviceProfile saveOrUpdate(EntitiesImportCtx ctx, DeviceProfile deviceProfile, EntityExportData<DeviceProfile> exportData, IdProvider idProvider) {
protected DeviceProfile saveOrUpdate(EntitiesImportCtx ctx, DeviceProfile deviceProfile, EntityExportData<DeviceProfile> exportData, IdProvider idProvider, CompareResult compareResult) {
DeviceProfile saved = deviceProfileService.saveDeviceProfile(deviceProfile);
if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) {
importCalculatedFields(ctx, saved, exportData, idProvider);

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java

@ -55,7 +55,7 @@ public class EntityViewImportService extends BaseEntityImportService<EntityViewI
}
@Override
protected EntityView saveOrUpdate(EntitiesImportCtx ctx, EntityView entityView, EntityExportData<EntityView> exportData, IdProvider idProvider) {
protected EntityView saveOrUpdate(EntitiesImportCtx ctx, EntityView entityView, EntityExportData<EntityView> exportData, IdProvider idProvider, CompareResult compareResult) {
return entityViewService.saveEntityView(entityView);
}

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationRuleImportService.java

@ -135,7 +135,7 @@ public class NotificationRuleImportService extends BaseEntityImportService<Notif
}
@Override
protected NotificationRule saveOrUpdate(EntitiesImportCtx ctx, NotificationRule notificationRule, EntityExportData<NotificationRule> exportData, IdProvider idProvider) {
protected NotificationRule saveOrUpdate(EntitiesImportCtx ctx, NotificationRule notificationRule, EntityExportData<NotificationRule> exportData, IdProvider idProvider, CompareResult compareResult) {
ConstraintValidator.validateFields(notificationRule);
return notificationRuleService.saveNotificationRule(ctx.getTenantId(), notificationRule);
}

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationTargetImportService.java

@ -80,7 +80,7 @@ public class NotificationTargetImportService extends BaseEntityImportService<Not
}
@Override
protected NotificationTarget saveOrUpdate(EntitiesImportCtx ctx, NotificationTarget notificationTarget, EntityExportData<NotificationTarget> exportData, IdProvider idProvider) {
protected NotificationTarget saveOrUpdate(EntitiesImportCtx ctx, NotificationTarget notificationTarget, EntityExportData<NotificationTarget> exportData, IdProvider idProvider, CompareResult compareResult) {
ConstraintValidator.validateFields(notificationTarget);
return notificationTargetService.saveNotificationTarget(ctx.getTenantId(), notificationTarget);
}

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationTemplateImportService.java

@ -48,7 +48,7 @@ public class NotificationTemplateImportService extends BaseEntityImportService<N
}
@Override
protected NotificationTemplate saveOrUpdate(EntitiesImportCtx ctx, NotificationTemplate notificationTemplate, EntityExportData<NotificationTemplate> exportData, IdProvider idProvider) {
protected NotificationTemplate saveOrUpdate(EntitiesImportCtx ctx, NotificationTemplate notificationTemplate, EntityExportData<NotificationTemplate> exportData, IdProvider idProvider, CompareResult compareResult) {
ConstraintValidator.validateFields(notificationTemplate);
return notificationTemplateService.saveNotificationTemplate(ctx.getTenantId(), notificationTemplate);
}

20
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java

@ -58,21 +58,29 @@ public class ResourceImportService extends BaseEntityImportService<TbResourceId,
}
@Override
protected boolean compare(EntitiesImportCtx ctx, EntityExportData<TbResource> exportData, TbResource prepared, TbResource existing) {
return true;
protected TbResource deepCopy(TbResource resource) {
return new TbResource(resource);
}
@Override
protected TbResource deepCopy(TbResource resource) {
return new TbResource(resource);
protected void cleanupForComparison(TbResource resource) {
super.cleanupForComparison(resource);
resource.setSearchText(null);
if (resource.getDescriptor().isNull()) {
resource.setDescriptor(null);
}
}
@Override
protected TbResource saveOrUpdate(EntitiesImportCtx ctx, TbResource resource, EntityExportData<TbResource> exportData, IdProvider idProvider) {
protected TbResource saveOrUpdate(EntitiesImportCtx ctx, TbResource resource, EntityExportData<TbResource> exportData, IdProvider idProvider, CompareResult compareResult) {
if (resource.getResourceType() == ResourceType.IMAGE) {
return new TbResource(imageService.saveImage(resource));
} else {
resource = resourceService.saveResource(resource);
if (compareResult.isExternalIdChangedOnly()) {
resource = resourceService.saveResource(resource, false);
} else {
resource = resourceService.saveResource(resource);
}
resource.setData(null);
resource.setPreview(null);
return resource;

14
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java

@ -18,6 +18,7 @@ package org.thingsboard.server.service.sync.ie.importing.impl;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.audit.ActionType;
@ -27,6 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.RuleChainExportData;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.rule.RuleNodeDao;
@ -103,7 +105,7 @@ public class RuleChainImportService extends BaseEntityImportService<RuleChainId,
}
@Override
protected RuleChain saveOrUpdate(EntitiesImportCtx ctx, RuleChain ruleChain, RuleChainExportData exportData, IdProvider idProvider) {
protected RuleChain saveOrUpdate(EntitiesImportCtx ctx, RuleChain ruleChain, RuleChainExportData exportData, IdProvider idProvider, CompareResult compareResult) {
ruleChain = ruleChainService.saveRuleChain(ruleChain);
if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) {
exportData.getMetaData().setRuleChainId(ruleChain.getId());
@ -115,15 +117,15 @@ public class RuleChainImportService extends BaseEntityImportService<RuleChainId,
}
@Override
protected boolean compare(EntitiesImportCtx ctx, RuleChainExportData exportData, RuleChain prepared, RuleChain existing) {
boolean different = super.compare(ctx, exportData, prepared, existing);
if (!different) {
protected boolean isUpdateNeeded(EntitiesImportCtx ctx, RuleChainExportData exportData, RuleChain prepared, RuleChain existing) {
boolean updateNeeded = super.isUpdateNeeded(ctx, exportData, prepared, existing);
if (!updateNeeded) {
RuleChainMetaData newMD = exportData.getMetaData();
RuleChainMetaData existingMD = ruleChainService.loadRuleChainMetaData(ctx.getTenantId(), prepared.getId());
existingMD.setRuleChainId(null);
different = !newMD.equals(existingMD);
updateNeeded = !newMD.equals(existingMD);
}
return different;
return updateNeeded;
}
@Override

6
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetTypeImportService.java

@ -44,13 +44,13 @@ public class WidgetTypeImportService extends BaseEntityImportService<WidgetTypeI
}
@Override
protected WidgetTypeDetails saveOrUpdate(EntitiesImportCtx ctx, WidgetTypeDetails widgetsBundle, WidgetTypeExportData exportData, IdProvider idProvider) {
protected WidgetTypeDetails saveOrUpdate(EntitiesImportCtx ctx, WidgetTypeDetails widgetsBundle, WidgetTypeExportData exportData, IdProvider idProvider, CompareResult compareResult) {
return widgetTypeService.saveWidgetType(widgetsBundle);
}
@Override
protected boolean compare(EntitiesImportCtx ctx, WidgetTypeExportData exportData, WidgetTypeDetails prepared, WidgetTypeDetails existing) {
return true;
protected CompareResult compare(EntitiesImportCtx ctx, WidgetTypeExportData exportData, WidgetTypeDetails prepared, WidgetTypeDetails existing) {
return new CompareResult(true);
}
@Override

6
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java

@ -49,7 +49,7 @@ public class WidgetsBundleImportService extends BaseEntityImportService<WidgetsB
}
@Override
protected WidgetsBundle saveOrUpdate(EntitiesImportCtx ctx, WidgetsBundle widgetsBundle, WidgetsBundleExportData exportData, IdProvider idProvider) {
protected WidgetsBundle saveOrUpdate(EntitiesImportCtx ctx, WidgetsBundle widgetsBundle, WidgetsBundleExportData exportData, IdProvider idProvider, CompareResult compareResult) {
if (CollectionsUtil.isNotEmpty(exportData.getWidgets())) {
exportData.getWidgets().forEach(widgetTypeNode -> {
String bundleAlias = widgetTypeNode.remove("bundleAlias").asText();
@ -75,8 +75,8 @@ public class WidgetsBundleImportService extends BaseEntityImportService<WidgetsB
}
@Override
protected boolean compare(EntitiesImportCtx ctx, WidgetsBundleExportData exportData, WidgetsBundle prepared, WidgetsBundle existing) {
return true;
protected CompareResult compare(EntitiesImportCtx ctx, WidgetsBundleExportData exportData, WidgetsBundle prepared, WidgetsBundle existing) {
return new CompareResult(true);
}
@Override

2
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java

@ -454,7 +454,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
}
private VersionLoadResult onError(EntityId externalId, Throwable e) {
return analyze(e, externalId).orElse(VersionLoadResult.error(EntityLoadError.runtimeError(e)));
return analyze(e, externalId).orElse(VersionLoadResult.error(EntityLoadError.runtimeError(e, externalId)));
}
private Optional<VersionLoadResult> analyze(Throwable e, EntityId externalId) {

2
application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java

@ -115,7 +115,7 @@ public class DefaultAlarmSubscriptionService extends AbstractSubscriptionService
}
@Override
public Boolean deleteAlarm(TenantId tenantId, AlarmId alarmId) {
public boolean deleteAlarm(TenantId tenantId, AlarmId alarmId) {
AlarmApiCallResult result = alarmService.delAlarm(tenantId, alarmId);
onAlarmDeleted(result);
return result.isSuccessful();

33
application/src/main/resources/thingsboard.yml

@ -644,6 +644,9 @@ cache:
mobileSecretKey:
timeToLiveInMinutes: "${CACHE_MOBILE_SECRET_KEY_TTL:2}" # QR secret key cache TTL
maxSize: "${CACHE_MOBILE_SECRET_KEY_MAX_SIZE:10000}" # 0 means the cache is disabled
trendzSettings:
timeToLiveInMinutes: "${CACHE_SPECS_TRENDZ_SETTINGS_TTL:1440}" # Trendz settings cache TTL
maxSize: "${CACHE_SPECS_TRENDZ_SETTINGS_MAX_SIZE:10000}" # 0 means the cache is disabled
# Deliberately placed outside the 'specs' group above
notificationRules:
@ -1628,6 +1631,12 @@ queue:
- key: max.poll.records
# Max poll records for edqs.state topic
value: "${TB_QUEUE_KAFKA_EDQS_STATE_MAX_POLL_RECORDS:512}"
# If you override any default Kafka topic name using environment variables, you must also specify the related consumer properties
# for the new topic in `consumer-properties-per-topic-inline`. Otherwise, the topic will not inherit its expected configuration (e.g., max.poll.records, timeouts, etc).
# Each entry sets a single property for a specific topic. To define multiple properties for a topic, repeat the topic key.
# Format: "topic1:key=value;topic1:key=value;topic2:key=value"
# Example: tb_core_updated:max.poll.records=10;tb_core_updated:bootstrap.servers=kafka1:9092,kafka2:9092;tb_edge_updated:auto.offset.reset=latest
consumer-properties-per-topic-inline: "${TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE:}"
other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000"
other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside
# - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms
@ -1952,3 +1961,27 @@ mobileApp:
googlePlayLink: "${TB_MOBILE_APP_GOOGLE_PLAY_LINK:https://play.google.com/store/apps/details?id=org.thingsboard.demo.app}"
# Link to App Store for Thingsboard Live mobile application
appStoreLink: "${TB_MOBILE_APP_APP_STORE_LINK:https://apps.apple.com/us/app/thingsboard-live/id1594355695}"
mqtt:
# MQTT client configuration parameters
client:
# Parameters that control the retransmission mechanism.
# This mechanism only applies to the handling of MQTT Publish, Subscribe, Unsubscribe and Pubrel messages.
# With the updated default settings:
# - After sending the message, wait approximately 5000 ms (± jitter) for the 1st attempt.
# - The 2nd attempt will occur after roughly 5000 * 2 = 10,000 ms (± jitter).
# - The 3rd attempt will occur after roughly 5000 * 4 = 20,000 ms (± jitter).
# - The 4th "attempt" will not actually perform a retransmission.
# Instead, the system will detect that the maximum number of attempts has been reached and drop the pending message.
retransmission:
# Maximum number of retransmission attempts allowed.
# If the attempt count exceeds this value, retransmissions will stop and the pending message will be dropped.
max_attempts: "${TB_MQTT_CLIENT_RETRANSMISSION_MAX_ATTEMPTS:3}"
# Base delay (in milliseconds) before the first retransmission attempt, measured from the moment the message is sent.
# Subsequent delays are calculated using exponential backoff.
# This base delay is also used as the reference value for applying jitter.
initial_delay_millis: "${TB_MQTT_CLIENT_RETRANSMISSION_INITIAL_DELAY_MILLIS:5000}"
# Jitter factor applied to the calculated retransmission delay.
# The actual delay is randomized within a range defined by multiplying the base delay by a factor between (1 - jitter_factor) and (1 + jitter_factor).
# For example, a jitter_factor of 0.15 means the actual delay may vary by up to ±15% of the base delay.
jitter_factor: "${TB_MQTT_CLIENT_RETRANSMISSION_JITTER_FACTOR:0.15}"

12
application/src/test/java/org/thingsboard/server/actors/rule/DefaultTbContextTest.java

@ -770,11 +770,11 @@ class DefaultTbContextTest {
ToRuleEngineMsg actualToRuleEngineMsg = toRuleEngineMsgCaptor.getValue();
assertThat(actualToRuleEngineMsg).usingRecursiveComparison()
.ignoringFields("tbMsg_")
.ignoringFields("tbMsgProto_")
.isEqualTo(ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits())
.setTenantIdLSB(TENANT_ID.getId().getLeastSignificantBits())
.setTbMsg(TbMsg.toByteString(expectedTbMsg))
.setTbMsgProto(TbMsg.toProto(expectedTbMsg))
.addAllRelationTypes(List.of(connectionType)).build());
var simpleTbQueueCallback = simpleTbQueueCallbackCaptor.getValue();
@ -827,11 +827,11 @@ class DefaultTbContextTest {
ToRuleEngineMsg actualToRuleEngineMsg = toRuleEngineMsgCaptor.getValue();
assertThat(actualToRuleEngineMsg).usingRecursiveComparison()
.ignoringFields("tbMsg_")
.ignoringFields("tbMsgProto_")
.isEqualTo(ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits())
.setTenantIdLSB(TENANT_ID.getId().getLeastSignificantBits())
.setTbMsg(TbMsg.toByteString(expectedTbMsg))
.setTbMsgProto(TbMsg.toProto(expectedTbMsg))
.build());
var simpleTbQueueCallback = simpleTbQueueCallbackCaptor.getValue();
@ -1009,11 +1009,11 @@ class DefaultTbContextTest {
ToRuleEngineMsg actualToRuleEngineMsg = toRuleEngineMsgCaptor.getValue();
assertThat(actualToRuleEngineMsg).usingRecursiveComparison()
.ignoringFields("tbMsg_")
.ignoringFields("tbMsgProto_.id_")
.isEqualTo(ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits())
.setTenantIdLSB(TENANT_ID.getId().getLeastSignificantBits())
.setTbMsg(TbMsg.toByteString(expectedTbMsg))
.setTbMsgProto(TbMsg.toProto(expectedTbMsg))
.setFailureMessage(EXCEPTION_MSG)
.addAllRelationTypes(List.of(TbNodeConnectionType.FAILURE)).build());

30
application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.controller;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.extern.slf4j.Slf4j;
@ -57,6 +58,8 @@ import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.is;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@Slf4j
@ -308,6 +311,21 @@ public class AlarmControllerTest extends AbstractControllerTest {
testNotifyEntityNever(alarm.getId(), alarm);
}
@Test
public void testDeleteNonExistentAlarm() throws Exception {
loginTenantAdmin();
var nonExistentAlarmId = Uuids.timeBased();
Mockito.reset(tbClusterService, auditLogService);
doDelete("/api/alarm/" + nonExistentAlarmId)
.andExpect(status().isNotFound())
.andExpect(statusReason(is("Alarm with id [" + nonExistentAlarmId + "] is not found")));
verifyNoInteractions(tbClusterService, auditLogService);
}
@Test
public void testClearAlarmViaCustomer() throws Exception {
loginCustomerUser();
@ -634,12 +652,12 @@ public class AlarmControllerTest extends AbstractControllerTest {
doDelete("/api/user/" + savedUser.getId().getId()).andExpect(status().isOk());
Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> {
AlarmInfo alarmInfo = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class);
Assert.assertNotNull(alarmInfo);
Assert.assertNull(alarmInfo.getAssigneeId());
Assert.assertTrue(alarmInfo.getAssignTs() >= afterAssignmentTs);
});
Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> {
AlarmInfo alarmInfo = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class);
Assert.assertNotNull(alarmInfo);
Assert.assertNull(alarmInfo.getAssigneeId());
Assert.assertTrue(alarmInfo.getAssignTs() >= afterAssignmentTs);
});
}
@Test

2
application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java

@ -216,7 +216,7 @@ public class OtaPackageControllerTest extends AbstractControllerTest {
Assert.assertEquals(CHECKSUM_ALGORITHM, savedFirmware.getChecksumAlgorithm().name());
Assert.assertEquals(CHECKSUM, savedFirmware.getChecksum());
testNotifyEntityAllOneTime(savedFirmware, savedFirmware.getId(), savedFirmware.getId(),
testNotifyEntityAllOneTime(new OtaPackageInfo(savedFirmware), savedFirmware.getId(), savedFirmware.getId(),
savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(),
ActionType.UPDATED);
}

4
application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java

File diff suppressed because one or more lines are too long

3
application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java

@ -762,7 +762,8 @@ public class TenantControllerTest extends AbstractControllerTest {
TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setTbMsg(TbMsg.toByteString(tbMsg)).build();
.setTbMsgProto(TbMsg.toProto(tbMsg))
.build();
tbClusterService.pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, null);
return tbMsg;
}

77
application/src/test/java/org/thingsboard/server/controller/TrendzControllerTest.java

@ -0,0 +1,77 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.controller;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.server.common.data.trendz.TrendzSettings;
import org.thingsboard.server.dao.service.DaoSqlTest;
import static org.assertj.core.api.Assertions.assertThat;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@DaoSqlTest
public class TrendzControllerTest extends AbstractControllerTest {
private final String trendzUrl = "https://some.domain.com:18888/also_necessary_prefix";
@Before
public void setUp() throws Exception {
loginTenantAdmin();
TrendzSettings trendzSettings = new TrendzSettings();
trendzSettings.setEnabled(true);
trendzSettings.setBaseUrl(trendzUrl);
doPost("/api/trendz/settings", trendzSettings).andExpect(status().isOk());
}
@Test
public void testTrendzSettingsWhenTenant() throws Exception {
loginTenantAdmin();
TrendzSettings trendzSettings = doGet("/api/trendz/settings", TrendzSettings.class);
assertThat(trendzSettings).isNotNull();
assertThat(trendzSettings.isEnabled()).isTrue();
assertThat(trendzSettings.getBaseUrl()).isEqualTo(trendzUrl);
String updatedUrl = "https://some.domain.com:18888/tenant_trendz";
trendzSettings.setBaseUrl(updatedUrl);
doPost("/api/trendz/settings", trendzSettings).andExpect(status().isOk());
TrendzSettings updatedTrendzSettings = doGet("/api/trendz/settings", TrendzSettings.class);
assertThat(updatedTrendzSettings).isEqualTo(trendzSettings);
}
@Test
public void testTrendzSettingsWhenCustomer() throws Exception {
loginCustomerUser();
TrendzSettings newTrendzSettings = new TrendzSettings();
newTrendzSettings.setEnabled(true);
newTrendzSettings.setBaseUrl("https://some.domain.com:18888/customer_trendz");
doPost("/api/trendz/settings", newTrendzSettings).andExpect(status().isForbidden());
TrendzSettings fetchedTrendzSettings = doGet("/api/trendz/settings", TrendzSettings.class);
assertThat(fetchedTrendzSettings).isNotNull();
assertThat(fetchedTrendzSettings.isEnabled()).isTrue();
assertThat(fetchedTrendzSettings.getBaseUrl()).isEqualTo(trendzUrl);
}
}

163
application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java

@ -15,15 +15,12 @@
*/
package org.thingsboard.server.service.entitiy.alarm;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.extern.slf4j.Slf4j;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.boot.test.mock.mockito.SpyBean;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.User;
@ -35,6 +32,9 @@ import org.thingsboard.server.common.data.alarm.AlarmInfo;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.AlarmId;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.dao.alarm.AlarmService;
@ -47,7 +47,6 @@ import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.service.entitiy.TbLogEntityActionService;
import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.security.permission.AccessControlService;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import org.thingsboard.server.service.telemetry.AlarmSubscriptionService;
@ -55,58 +54,57 @@ import org.thingsboard.server.service.telemetry.AlarmSubscriptionService;
import java.util.List;
import java.util.UUID;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
@Slf4j
@RunWith(SpringRunner.class)
@ContextConfiguration(classes = DefaultTbAlarmService.class)
@TestPropertySource(properties = {
"server.log_controller_error_stack_trace=false"
})
public class DefaultTbAlarmServiceTest {
@SpringJUnitConfig(DefaultTbAlarmService.class)
class DefaultTbAlarmServiceTest {
@MockBean
protected DbCallbackExecutorService dbExecutor;
TbLogEntityActionService logEntityActionService;
@MockBean
protected TbLogEntityActionService logEntityActionService;
EdgeService edgeService;
@MockBean
protected EdgeService edgeService;
AlarmService alarmService;
@MockBean
protected AlarmService alarmService;
TbAlarmCommentService alarmCommentService;
@MockBean
protected TbAlarmCommentService alarmCommentService;
AlarmSubscriptionService alarmSubscriptionService;
@MockBean
protected AlarmSubscriptionService alarmSubscriptionService;
CustomerService customerService;
@MockBean
protected CustomerService customerService;
TbClusterService tbClusterService;
@MockBean
protected TbClusterService tbClusterService;
EntitiesVersionControlService vcService;
@MockBean
private EntitiesVersionControlService vcService;
AccessControlService accessControlService;
@MockBean
private AccessControlService accessControlService;
TenantService tenantService;
@MockBean
private TenantService tenantService;
AssetService assetService;
@MockBean
private AssetService assetService;
DeviceService deviceService;
@MockBean
private DeviceService deviceService;
AssetProfileService assetProfileService;
@MockBean
private AssetProfileService assetProfileService;
DeviceProfileService deviceProfileService;
@MockBean
private DeviceProfileService deviceProfileService;
@MockBean
private EntityService entityService;
@SpyBean
EntityService entityService;
@Autowired
DefaultTbAlarmService service;
TenantId tenantId = TenantId.fromUUID(Uuids.timeBased());
CustomerId customerId = new CustomerId(Uuids.timeBased());
@Test
public void testSave() throws ThingsboardException {
void testSave() throws ThingsboardException {
var alarm = new AlarmInfo();
when(alarmSubscriptionService.createAlarm(any())).thenReturn(AlarmApiCallResult.builder()
.successful(true)
@ -115,45 +113,99 @@ public class DefaultTbAlarmServiceTest {
.build());
service.save(alarm, new User());
verify(logEntityActionService, times(1)).logEntityAction(any(), any(), any(), any(), eq(ActionType.ADDED), any());
verify(alarmSubscriptionService, times(1)).createAlarm(any());
verify(logEntityActionService).logEntityAction(any(), any(), any(), any(), eq(ActionType.ADDED), any());
verify(alarmSubscriptionService).createAlarm(any());
}
@Test
public void testAck() throws ThingsboardException {
void testAck() throws ThingsboardException {
var alarm = new Alarm();
when(alarmSubscriptionService.acknowledgeAlarm(any(), any(), anyLong()))
.thenReturn(AlarmApiCallResult.builder().successful(true).modified(true).alarm(new AlarmInfo()).build());
service.ack(alarm, new User(new UserId(UUID.randomUUID())));
verify(alarmCommentService, times(1)).saveAlarmComment(any(), any(), any());
verify(logEntityActionService, times(1)).logEntityAction(any(), any(), any(), any(), eq(ActionType.ALARM_ACK), any());
verify(alarmSubscriptionService, times(1)).acknowledgeAlarm(any(), any(), anyLong());
verify(alarmCommentService).saveAlarmComment(any(), any(), any());
verify(logEntityActionService).logEntityAction(any(), any(), any(), any(), eq(ActionType.ALARM_ACK), any());
verify(alarmSubscriptionService).acknowledgeAlarm(any(), any(), anyLong());
}
@Test
public void testClear() throws ThingsboardException {
void testClear() throws ThingsboardException {
var alarm = new Alarm();
alarm.setAcknowledged(true);
when(alarmSubscriptionService.clearAlarm(any(), any(), anyLong(), any()))
.thenReturn(AlarmApiCallResult.builder().successful(true).cleared(true).alarm(new AlarmInfo()).build());
service.clear(alarm, new User(new UserId(UUID.randomUUID())));
verify(alarmCommentService, times(1)).saveAlarmComment(any(), any(), any());
verify(logEntityActionService, times(1)).logEntityAction(any(), any(), any(), any(), eq(ActionType.ALARM_CLEAR), any());
verify(alarmSubscriptionService, times(1)).clearAlarm(any(), any(), anyLong(), any());
verify(alarmCommentService).saveAlarmComment(any(), any(), any());
verify(logEntityActionService).logEntityAction(any(), any(), any(), any(), eq(ActionType.ALARM_CLEAR), any());
verify(alarmSubscriptionService).clearAlarm(any(), any(), anyLong(), any());
}
@Test
public void testDelete() {
service.delete(new Alarm(), new User());
void testDelete_deleteApiReturnsTrue_shouldLogActionAndReturnTrue() {
// GIVEN
var alarmOriginator = new DeviceId(Uuids.timeBased());
var alarm = new Alarm(new AlarmId(Uuids.timeBased()));
alarm.setTenantId(tenantId);
alarm.setCustomerId(customerId);
alarm.setOriginator(alarmOriginator);
var user = new User();
when(alarmSubscriptionService.deleteAlarm(tenantId, alarm.getId())).thenReturn(true);
verify(logEntityActionService, times(1)).logEntityAction(any(), any(), any(), any(), eq(ActionType.ALARM_DELETE), any(), any());
verify(alarmSubscriptionService, times(1)).deleteAlarm(any(), any());
// WHEN
boolean actual = service.delete(alarm, user);
assertThat(actual).isTrue();
verify(logEntityActionService).logEntityAction(tenantId, alarmOriginator, alarm, alarm.getCustomerId(), ActionType.ALARM_DELETE, user, alarm.getId());
verify(alarmSubscriptionService).deleteAlarm(tenantId, alarm.getId());
}
@Test
public void testUnassignAlarm() throws ThingsboardException {
void testDelete_deleteApiReturnsFalse_shouldNotLogActionAndReturnFalse() {
// GIVEN
var alarm = new Alarm(new AlarmId(Uuids.timeBased()));
alarm.setTenantId(tenantId);
var user = new User();
// WHEN
boolean actual = service.delete(alarm, user);
assertThat(actual).isFalse();
verifyNoInteractions(logEntityActionService);
verify(alarmSubscriptionService).deleteAlarm(tenantId, alarm.getId());
}
@Test
void testDelete_deleteApiThrowsException_shouldLogFailedActionAndRethrow() {
// GIVEN
var alarmOriginator = new DeviceId(Uuids.timeBased());
var alarm = new Alarm(new AlarmId(Uuids.timeBased()));
alarm.setTenantId(tenantId);
alarm.setOriginator(alarmOriginator);
var user = new User();
var exception = new RuntimeException("failed to delete alarm");
when(alarmSubscriptionService.deleteAlarm(tenantId, alarm.getId())).thenThrow(exception);
// WHEN-THEN
assertThatThrownBy(() -> service.delete(alarm, user))
.isInstanceOf(RuntimeException.class)
.hasMessage("failed to delete alarm");
verify(logEntityActionService).logEntityAction(tenantId, new DeviceId(EntityId.NULL_UUID), ActionType.ALARM_DELETE, user, exception, alarm.getId());
verify(alarmSubscriptionService).deleteAlarm(tenantId, alarm.getId());
}
@Test
void testUnassignAlarm() throws ThingsboardException {
AlarmInfo alarm = new AlarmInfo();
alarm.setId(new AlarmId(UUID.randomUUID()));
when(alarmSubscriptionService.unassignAlarm(any(), any(), anyLong()))
@ -174,12 +226,11 @@ public class DefaultTbAlarmServiceTest {
.comment(commentNode)
.build();
verify(alarmCommentService, times(1))
.saveAlarmComment(eq(alarm), eq(expectedAlarmComment), eq(user));
verify(alarmCommentService).saveAlarmComment(eq(alarm), eq(expectedAlarmComment), eq(user));
}
@Test
public void testUnassignDeletedUserAlarms() throws ThingsboardException {
void testUnassignDeletedUserAlarms() throws ThingsboardException {
AlarmInfo alarm = new AlarmInfo();
alarm.setId(new AlarmId(UUID.randomUUID()));
@ -189,7 +240,7 @@ public class DefaultTbAlarmServiceTest {
User user = new User();
user.setEmail("testEmail@gmail.com");
user.setId(new UserId(UUID.randomUUID()));
service.unassignDeletedUserAlarms(new TenantId(UUID.randomUUID()), user.getId(), user.getTitle(), List.of(alarm.getUuidId()), System.currentTimeMillis());
service.unassignDeletedUserAlarms(tenantId, user.getId(), user.getTitle(), List.of(alarm.getUuidId()), System.currentTimeMillis());
ObjectNode commentNode = JacksonUtil.newObjectNode();
commentNode.put("subtype", "ASSIGN");
@ -200,9 +251,7 @@ public class DefaultTbAlarmServiceTest {
.comment(commentNode)
.build();
verify(alarmCommentService, times(1))
.saveAlarmComment(eq(alarm), eq(expectedAlarmComment), eq(null));
verify(alarmCommentService).saveAlarmComment(eq(alarm), eq(expectedAlarmComment), eq(null));
}
}

2
application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java

@ -737,7 +737,7 @@ public class TbRuleEngineQueueConsumerManagerTest {
.setTenantIdMSB(tenantId.getMostSignificantBits())
.setTenantIdLSB(tenantId.getLeastSignificantBits())
.addRelationTypes("Success")
.setTbMsg(TbMsg.toByteString(tbMsg))
.setTbMsgProto(TbMsg.toProto(tbMsg))
.build());
}

2
application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineStrategyTest.java

@ -263,7 +263,7 @@ public class TbRuleEngineStrategyTest {
.setTenantIdMSB(tenantId.getMostSignificantBits())
.setTenantIdLSB(tenantId.getLeastSignificantBits())
.addRelationTypes("Success")
.setTbMsg(TbMsg.toByteString(tbMsg))
.setTbMsgProto(TbMsg.toProto(tbMsg))
.build());
}

2
application/src/test/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcServiceTest.java

@ -55,7 +55,7 @@ class DefaultTbRuleEngineRpcServiceTest {
var restApiCallResponseMsgProto = TransportProtos.RestApiCallResponseMsgProto.newBuilder()
.setRequestIdMSB(requestId.getMostSignificantBits())
.setRequestIdLSB(requestId.getLeastSignificantBits())
.setResponse(TbMsg.toByteString(msg))
.setResponseProto(TbMsg.toProto(msg))
.build();
// WHEN

2
application/src/test/java/org/thingsboard/server/service/ruleengine/DefaultRuleEngineCallServiceTest.java

@ -144,7 +144,7 @@ public class DefaultRuleEngineCallServiceTest {
private TransportProtos.RestApiCallResponseMsgProto getResponse(UUID requestId, TbMsg msg) {
return TransportProtos.RestApiCallResponseMsgProto.newBuilder()
.setResponse(TbMsg.toByteString(msg))
.setResponseProto(TbMsg.toProto(msg))
.setRequestIdMSB(requestId.getMostSignificantBits())
.setRequestIdLSB(requestId.getLeastSignificantBits())
.build();

43
application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java

@ -40,6 +40,9 @@ import org.thingsboard.server.common.data.EntityView;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.HasTenantId;
import org.thingsboard.server.common.data.OtaPackage;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.TbResourceInfo;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.asset.Asset;
@ -113,6 +116,8 @@ import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import static org.thingsboard.server.controller.TbResourceControllerTest.TEST_DATA;
import static org.thingsboard.server.controller.TbResourceControllerTest.JS_TEST_FILE_NAME;
@DaoSqlTest
public class VersionControlTest extends AbstractControllerTest {
@ -648,6 +653,26 @@ public class VersionControlTest extends AbstractControllerTest {
assertThat(importedCalculatedField.getType()).isEqualTo(calculatedField.getType());
}
@Test
public void testResourceVc_sameTenant() throws Exception {
TbResourceInfo resourceInfo = createResource("Test resource");
String versionId = createVersion("resources", EntityType.TB_RESOURCE);
TbResource resource = findResource(resourceInfo.getName());
loadVersion(versionId, EntityType.TB_RESOURCE);
TbResource importedResource = findResource(resource.getName());
checkImportedEntity(tenantId1, resource, tenantId1, importedResource);
checkImportedResourceData(resource, importedResource);
}
protected void checkImportedResourceData(TbResource resource, TbResource importedResource) {
assertThat(importedResource.getName()).isEqualTo(resource.getName());
assertThat(importedResource.getData()).isEqualTo(resource.getData());
assertThat(importedResource.getResourceKey()).isEqualTo(resource.getResourceKey());
assertThat(importedResource.getResourceType()).isEqualTo(resource.getResourceType());
}
private <E extends ExportableEntity<?> & HasTenantId> void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, E importedEntity) {
assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1);
assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2);
@ -1128,4 +1153,22 @@ public class VersionControlTest extends AbstractControllerTest {
return doGetTypedWithPageLink("/api/" + entityId.getEntityType() + "/" + entityId.getId() + "/calculatedFields?", new TypeReference<PageData<CalculatedField>>() {}, new PageLink(100, 0)).getData();
}
private TbResourceInfo createResource(String name) {
TbResource resource = new TbResource();
resource.setResourceType(ResourceType.JKS);
resource.setTitle(name);
resource.setFileName(JS_TEST_FILE_NAME);
resource.setEncodedData(TEST_DATA);
return saveTbResource(resource);
}
private TbResourceInfo saveTbResource(TbResource tbResource) {
return doPost("/api/resource", tbResource, TbResourceInfo.class);
}
private TbResource findResource(String name) throws Exception {
return doGetTypedWithPageLink("/api/resource?", new TypeReference<PageData<TbResource>>() {}, new PageLink(100, 0, name)).getData().get(0);
}
}

29
common/dao-api/src/main/java/org/thingsboard/server/dao/trendz/TrendzSettingsService.java

@ -0,0 +1,29 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.trendz;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.trendz.TrendzSettings;
public interface TrendzSettingsService {
void saveTrendzSettings(TenantId tenantId, TrendzSettings settings);
TrendzSettings findTrendzSettings(TenantId tenantId);
void deleteTrendzSettings(TenantId tenantId);
}

1
common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java

@ -35,6 +35,7 @@ public class CacheConstants {
public static final String DEVICE_PROFILE_CACHE = "deviceProfiles";
public static final String NOTIFICATION_SETTINGS_CACHE = "notificationSettings";
public static final String SENT_NOTIFICATIONS_CACHE = "sentNotifications";
public static final String TRENDZ_SETTINGS_CACHE = "trendzSettings";
public static final String ASSET_PROFILE_CACHE = "assetProfiles";
public static final String ATTRIBUTES_CACHE = "attributes";

2
common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java

@ -17,6 +17,7 @@ package org.thingsboard.server.common.data;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.Data;
import org.thingsboard.server.common.data.trendz.TrendzSettings;
import java.util.List;
@ -37,4 +38,5 @@ public class SystemParams {
String calculatedFieldDebugPerTenantLimitsConfiguration;
long maxArgumentsPerCF;
long maxDataPointsPerRollingArg;
TrendzSettings trendzSettings;
}

8
common/data/src/main/java/org/thingsboard/server/common/data/TbProperty.java

@ -15,14 +15,16 @@
*/
package org.thingsboard.server.common.data;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* Created by ashvayka on 25.09.18.
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class TbProperty {
private String key;
private String value;
}

1
common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java

@ -43,6 +43,7 @@ public class EntityRelation implements HasVersion, Serializable, EdqsObject {
public static final String EDGE_TYPE = "ManagedByEdge";
public static final String CONTAINS_TYPE = "Contains";
public static final String MANAGES_TYPE = "Manages";
public static final String USES_TYPE = "Uses";
@Setter
private EntityId from;

6
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java

@ -45,11 +45,15 @@ public class EntityLoadError implements Serializable {
}
public static EntityLoadError runtimeError(Throwable e) {
return runtimeError(e, null);
}
public static EntityLoadError runtimeError(Throwable e, EntityId externalId) {
String message = e.getMessage();
if (StringUtils.isEmpty(message)) {
message = "unexpected error (" + ClassUtils.getShortClassName(e.getClass()) + ")";
}
return EntityLoadError.builder().type("RUNTIME").message(message).build();
return EntityLoadError.builder().type("RUNTIME").message(message).source(externalId).build();
}
}

26
common/data/src/main/java/org/thingsboard/server/common/data/trendz/TrendzSettings.java

@ -0,0 +1,26 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.trendz;
import lombok.Data;
@Data
public class TrendzSettings {
private boolean enabled;
private String baseUrl;
}

104
common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java

@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.msg.gen.MsgProtos;
import org.thingsboard.server.common.msg.gen.MsgProtos.TbMsgProto;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import java.io.Serializable;
@ -151,12 +152,8 @@ public final class TbMsg implements Serializable {
this.callback = Objects.requireNonNullElse(callback, TbMsgCallback.EMPTY);
}
public static ByteString toByteString(TbMsg msg) {
return ByteString.copyFrom(toByteArray(msg));
}
public static byte[] toByteArray(TbMsg msg) {
MsgProtos.TbMsgProto.Builder builder = MsgProtos.TbMsgProto.newBuilder();
public static TbMsgProto toProto(TbMsg msg) {
TbMsgProto.Builder builder = TbMsgProto.newBuilder();
builder.setId(msg.getId().toString());
builder.setTs(msg.getTs());
builder.setType(msg.getType());
@ -205,56 +202,55 @@ public final class TbMsg implements Serializable {
}
builder.setCtx(msg.ctx.toProto());
return builder.build().toByteArray();
return builder.build();
}
public static TbMsg fromBytes(String queueName, byte[] data, TbMsgCallback callback) {
@Deprecated(forRemoval = true, since = "4.1") // to be removed in 4.2
public static TbMsg fromProto(String queueName, TbMsgProto proto, ByteString data, TbMsgCallback callback) {
try {
MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(data);
TbMsgMetaData metaData = new TbMsgMetaData(proto.getMetaData().getDataMap());
EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()));
CustomerId customerId = null;
RuleChainId ruleChainId = null;
RuleNodeId ruleNodeId = null;
UUID correlationId = null;
Integer partition = null;
List<CalculatedFieldId> calculatedFieldIds = new CopyOnWriteArrayList<>();
if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) {
customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB()));
}
if (proto.getRuleChainIdMSB() != 0L && proto.getRuleChainIdLSB() != 0L) {
ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB()));
}
if (proto.getRuleNodeIdMSB() != 0L && proto.getRuleNodeIdLSB() != 0L) {
ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB()));
}
if (proto.getCorrelationIdMSB() != 0L && proto.getCorrelationIdLSB() != 0L) {
correlationId = new UUID(proto.getCorrelationIdMSB(), proto.getCorrelationIdLSB());
partition = proto.getPartition();
}
for (MsgProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) {
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(
cfIdProto.getCalculatedFieldIdMSB(),
cfIdProto.getCalculatedFieldIdLSB()
));
calculatedFieldIds.add(calculatedFieldId);
if (!data.isEmpty()) {
proto = TbMsgProto.parseFrom(data);
}
TbMsgProcessingCtx ctx;
if (proto.hasCtx()) {
ctx = TbMsgProcessingCtx.fromProto(proto.getCtx());
} else {
// Backward compatibility with unprocessed messages fetched from queue after update.
ctx = new TbMsgProcessingCtx(proto.getRuleNodeExecCounter());
}
TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()];
return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), null, proto.getType(), entityId, customerId,
metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback);
} catch (InvalidProtocolBufferException e) {
throw new IllegalStateException("Could not parse protobuf for TbMsg", e);
}
return fromProto(queueName, proto, callback);
}
public static TbMsg fromProto(String queueName, TbMsgProto proto, TbMsgCallback callback) {
TbMsgMetaData metaData = new TbMsgMetaData(proto.getMetaData().getDataMap());
EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()));
CustomerId customerId = null;
RuleChainId ruleChainId = null;
RuleNodeId ruleNodeId = null;
UUID correlationId = null;
Integer partition = null;
List<CalculatedFieldId> calculatedFieldIds = new CopyOnWriteArrayList<>();
if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) {
customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB()));
}
if (proto.getRuleChainIdMSB() != 0L && proto.getRuleChainIdLSB() != 0L) {
ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB()));
}
if (proto.getRuleNodeIdMSB() != 0L && proto.getRuleNodeIdLSB() != 0L) {
ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB()));
}
if (proto.getCorrelationIdMSB() != 0L && proto.getCorrelationIdLSB() != 0L) {
correlationId = new UUID(proto.getCorrelationIdMSB(), proto.getCorrelationIdLSB());
partition = proto.getPartition();
}
for (MsgProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) {
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(
cfIdProto.getCalculatedFieldIdMSB(),
cfIdProto.getCalculatedFieldIdLSB()
));
calculatedFieldIds.add(calculatedFieldId);
}
TbMsgProcessingCtx ctx = TbMsgProcessingCtx.fromProto(proto.getCtx());
TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()];
return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), null, proto.getType(), entityId, customerId,
metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback);
}
public int getAndIncrementRuleNodeCounter() {
@ -504,11 +500,11 @@ public final class TbMsg implements Serializable {
public String toString() {
return "TbMsg.TbMsgBuilder(queueName=" + this.queueName + ", id=" + this.id + ", ts=" + this.ts +
", type=" + this.type + ", internalType=" + this.internalType + ", originator=" + this.originator +
", customerId=" + this.customerId + ", metaData=" + this.metaData + ", dataType=" + this.dataType +
", data=" + this.data + ", ruleChainId=" + this.ruleChainId + ", ruleNodeId=" + this.ruleNodeId +
", correlationId=" + this.correlationId + ", partition=" + this.partition + ", previousCalculatedFields=" + this.previousCalculatedFieldIds +
", ctx=" + this.ctx + ", callback=" + this.callback + ")";
", type=" + this.type + ", internalType=" + this.internalType + ", originator=" + this.originator +
", customerId=" + this.customerId + ", metaData=" + this.metaData + ", dataType=" + this.dataType +
", data=" + this.data + ", ruleChainId=" + this.ruleChainId + ", ruleNodeId=" + this.ruleNodeId +
", correlationId=" + this.correlationId + ", partition=" + this.partition + ", previousCalculatedFields=" + this.previousCalculatedFieldIds +
", ctx=" + this.ctx + ", callback=" + this.callback + ")";
}
}

4
common/message/src/main/proto/tbmsg.proto

@ -59,8 +59,8 @@ message TbMsgProto {
string data = 14;
int64 ts = 15;
// Will be removed in 3.4. Moved to processing context
int32 ruleNodeExecCounter = 16;
// ruleNodeExecCounter (16) was removed in 4.1
int64 customerIdMSB = 17;
int64 customerIdLSB = 18;

22
common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java

@ -18,6 +18,8 @@ package org.thingsboard.server.common.util;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
@ -76,12 +78,15 @@ import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.DeviceCredentialsType;
import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg;
import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg;
import org.thingsboard.server.common.msg.edge.EdgeHighPriorityMsg;
import org.thingsboard.server.common.msg.edge.FromEdgeSyncResponse;
import org.thingsboard.server.common.msg.edge.ToEdgeSyncRequest;
import org.thingsboard.server.common.msg.gen.MsgProtos;
import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponseActorMsg;
import org.thingsboard.server.common.msg.rpc.RemoveRpcActorMsg;
@ -93,8 +98,8 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceDeleteMsg;
import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg;
import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto;
import org.thingsboard.server.gen.transport.TransportProtos.ApiUsageRecordKeyProto;
import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto;
import java.util.ArrayList;
import java.util.Arrays;
@ -1344,6 +1349,21 @@ public class ProtoUtils {
return builder.build();
}
@Deprecated(forRemoval = true, since = "4.1")
public static MsgProtos.TbMsgProto getTbMsgProto(TransportProtos.ToRuleEngineMsg ruleEngineMsg) throws InvalidProtocolBufferException {
if (ruleEngineMsg.getTbMsg().isEmpty()) {
return ruleEngineMsg.getTbMsgProto();
} else {
return MsgProtos.TbMsgProto.parseFrom(ruleEngineMsg.getTbMsg());
}
}
@SneakyThrows
@Deprecated(forRemoval = true, since = "4.1") // inline to TbMsg.fromProto(queueName, ruleEngineMsg.getTbMsgProto(), callback)
public static TbMsg fromTbMsgProto(String queueName, TransportProtos.ToRuleEngineMsg ruleEngineMsg, TbMsgCallback callback) {
return TbMsg.fromProto(queueName, getTbMsgProto(ruleEngineMsg), callback);
}
private static boolean isNotNull(Object obj) {
return obj != null;
}

8
common/proto/src/main/proto/queue.proto

@ -20,6 +20,8 @@ package transport;
option java_package = "org.thingsboard.server.gen.transport";
option java_outer_classname = "TransportProtos";
import "tbmsg.proto";
/**
* Common data structures
*/
@ -125,7 +127,8 @@ message SessionInfoProto {
message RestApiCallResponseMsgProto {
int64 requestIdMSB = 1;
int64 requestIdLSB = 2;
bytes response = 5;
bytes response = 5 [deprecated = true];
msgqueue.TbMsgProto responseProto = 6;
}
enum SessionEvent {
@ -1700,9 +1703,10 @@ message ToCalculatedFieldNotificationMsg {
message ToRuleEngineMsg {
int64 tenantIdMSB = 1;
int64 tenantIdLSB = 2;
bytes tbMsg = 3;
bytes tbMsg = 3 [deprecated = true]; // for removal in 4.2
repeated string relationTypes = 4;
string failureMessage = 5;
msgqueue.TbMsgProto tbMsgProto = 6;
}
message ToRuleEngineNotificationMsg {

2
common/queue/src/main/java/org/thingsboard/server/queue/common/TbRuleEngineProducerService.java

@ -65,7 +65,7 @@ public class TbRuleEngineProducerService {
log.trace("[{}][{}] Pushing to topic {} message {}", tenantId, tbMsg.getOriginator(), tpi.getFullTopicName(), tbMsg);
}
ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder()
.setTbMsg(TbMsg.toByteString(tbMsg))
.setTbMsgProto(TbMsg.toProto(tbMsg))
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build();
producer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback);

38
common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.queue.kafka;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.Getter;
import lombok.Setter;
@ -36,7 +37,8 @@ import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.TbProperty;
import org.thingsboard.server.queue.util.PropertyUtils;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@ -138,15 +140,26 @@ public class TbKafkaSettings {
@Value("${queue.kafka.other-inline:}")
private String otherInline;
@Value("${queue.kafka.consumer-properties-per-topic-inline:}")
private String consumerPropertiesPerTopicInline;
@Deprecated
@Setter
private List<TbProperty> other;
@Setter
private Map<String, List<TbProperty>> consumerPropertiesPerTopic = Collections.emptyMap();
private Map<String, List<TbProperty>> consumerPropertiesPerTopic = new HashMap<>();
private volatile AdminClient adminClient;
@PostConstruct
public void initInlineTopicProperties() {
Map<String, List<TbProperty>> inlineProps = parseTopicPropertyList(consumerPropertiesPerTopicInline);
if (!inlineProps.isEmpty()) {
consumerPropertiesPerTopic.putAll(inlineProps);
}
}
public Properties toConsumerProps(String topic) {
Properties props = toProps();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
@ -245,6 +258,27 @@ public class TbKafkaSettings {
return props;
}
private Map<String, List<TbProperty>> parseTopicPropertyList(String inlineProperties) {
Map<String, List<String>> grouped = PropertyUtils.getGroupedProps(inlineProperties);
Map<String, List<TbProperty>> result = new HashMap<>();
grouped.forEach((topic, entries) -> {
Map<String, String> merged = new LinkedHashMap<>();
for (String entry : entries) {
String[] kv = entry.split("=", 2);
if (kv.length == 2) {
merged.put(kv[0].trim(), kv[1].trim());
}
}
List<TbProperty> props = merged.entrySet().stream()
.map(e -> new TbProperty(e.getKey(), e.getValue()))
.toList();
result.put(topic, props);
});
return result;
}
@PreDestroy
private void destroy() {
if (adminClient != null) {

17
common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java

@ -17,7 +17,9 @@ package org.thingsboard.server.queue.util;
import org.thingsboard.server.common.data.StringUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
@ -38,6 +40,21 @@ public class PropertyUtils {
return configs;
}
public static Map<String, List<String>> getGroupedProps(String properties) {
Map<String, List<String>> configs = new HashMap<>();
if (StringUtils.isNotEmpty(properties)) {
for (String property : properties.split(";")) {
if (StringUtils.isNotEmpty(property)) {
int delimiterPosition = property.indexOf(":");
String topic = property.substring(0, delimiterPosition).trim();
String value = property.substring(delimiterPosition + 1).trim();
configs.computeIfAbsent(topic, k -> new ArrayList<>()).add(value);
}
}
}
return configs;
}
public static Map<String, String> getProps(Map<String, String> defaultProperties, String propertiesStr) {
return getProps(defaultProperties, propertiesStr, PropertyUtils::getProps);
}

20
common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java

@ -33,6 +33,12 @@ import static org.mockito.Mockito.spy;
"queue.type=kafka",
"queue.kafka.bootstrap.servers=localhost:9092",
"queue.kafka.other-inline=metrics.recording.level:INFO;metrics.sample.window.ms:30000",
"queue.kafka.consumer-properties-per-topic-inline=" +
"tb_core_updated:max.poll.records=10;" +
"tb_core_updated:enable.auto.commit=true;" +
"tb_core_updated:bootstrap.servers=kafka1:9092,kafka2:9092;" +
"tb_edge_updated:max.poll.records=5;" +
"tb_edge_updated:auto.offset.reset=latest"
})
class TbKafkaSettingsTest {
@ -79,4 +85,16 @@ class TbKafkaSettingsTest {
Mockito.verify(settings).configureSSL(any());
}
}
@Test
void givenMultipleTopicsInInlineConfig_whenParsed_thenEachTopicGetsExpectedProperties() {
Properties coreProps = settings.toConsumerProps("tb_core_updated");
assertThat(coreProps.getProperty("max.poll.records")).isEqualTo("10");
assertThat(coreProps.getProperty("enable.auto.commit")).isEqualTo("true");
assertThat(coreProps.getProperty("bootstrap.servers")).isEqualTo("kafka1:9092,kafka2:9092");
Properties edgeProps = settings.toConsumerProps("tb_edge_updated");
assertThat(edgeProps.getProperty("max.poll.records")).isEqualTo("5");
assertThat(edgeProps.getProperty("auto.offset.reset")).isEqualTo("latest");
}
}

37
dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java

@ -76,6 +76,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -204,11 +205,17 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC
}
}
}
RuleChainId ruleChainId = ruleChain.getId();
List<RuleNodeUpdateResult> updatedRuleNodes = new ArrayList<>();
List<RuleNode> existingRuleNodes = getRuleChainNodes(tenantId, ruleChainMetaData.getRuleChainId());
for (RuleNode existingNode : existingRuleNodes) {
relationService.deleteEntityRelations(tenantId, existingNode.getId());
if (existingNode.getType().equals(TB_RULE_CHAIN_INPUT_NODE)) {
EntityRelation relation = getRuleChainInputRelation(ruleChainId, existingNode);
if (relation != null) {
relationService.deleteRelation(tenantId, relation);
}
}
Integer index = ruleNodeIndexMap.get(existingNode.getId());
RuleNode newRuleNode = null;
if (index != null) {
@ -220,7 +227,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC
}
updatedRuleNodes.add(new RuleNodeUpdateResult(existingNode, newRuleNode));
}
RuleChainId ruleChainId = ruleChain.getId();
if (nodes != null) {
long now = System.currentTimeMillis();
for (RuleNode node : toAddOrUpdate) {
@ -233,6 +240,12 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC
RuleNode savedNode = ruleNodeDao.save(tenantId, node);
relations.add(new EntityRelation(ruleChainMetaData.getRuleChainId(), savedNode.getId(),
EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN));
if (node.getType().equals(TB_RULE_CHAIN_INPUT_NODE)) {
EntityRelation relation = getRuleChainInputRelation(ruleChainId, node);
if (relation != null) {
relations.add(relation);
}
}
int index = nodes.indexOf(node);
nodes.set(index, savedNode);
ruleNodeIndexMap.put(savedNode.getId(), index);
@ -247,7 +260,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC
firstRuleNodeId = nodes.get(ruleChainMetaData.getFirstNodeIndex()).getId();
}
if ((ruleChain.getFirstRuleNodeId() != null && !ruleChain.getFirstRuleNodeId().equals(firstRuleNodeId))
|| (ruleChain.getFirstRuleNodeId() == null && firstRuleNodeId != null)) {
|| (ruleChain.getFirstRuleNodeId() == null && firstRuleNodeId != null)) {
ruleChain.setFirstRuleNodeId(firstRuleNodeId);
}
if (ruleChainMetaData.getConnections() != null) {
@ -265,7 +278,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC
RuleNode targetNode = new RuleNode();
targetNode.setName(targetRuleChain != null ? targetRuleChain.getName() : "Rule Chain Input");
targetNode.setRuleChainId(ruleChainId);
targetNode.setType("org.thingsboard.rule.engine.flow.TbRuleChainInputNode");
targetNode.setType(TB_RULE_CHAIN_INPUT_NODE);
var configuration = JacksonUtil.newObjectNode();
configuration.put("ruleChainId", targetRuleChainId.getId().toString());
targetNode.setConfiguration(configuration);
@ -302,6 +315,22 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC
return RuleChainUpdateResult.successful(updatedRuleNodes);
}
private EntityRelation getRuleChainInputRelation(RuleChainId ruleChainId, RuleNode inputNode) {
RuleChainId targetRuleChainId = Optional.ofNullable(inputNode.getConfiguration().get("ruleChainId"))
.filter(JsonNode::isTextual).map(JsonNode::asText).map(id -> new RuleChainId(UUID.fromString(id)))
.orElse(null);
if (targetRuleChainId != null) {
EntityRelation relation = new EntityRelation();
relation.setFrom(ruleChainId);
relation.setTo(targetRuleChainId);
relation.setType(EntityRelation.USES_TYPE);
relation.setTypeGroup(RelationTypeGroup.COMMON);
return relation;
} else {
return null;
}
}
@Override
public RuleChainMetaData loadRuleChainMetaData(TenantId tenantId, RuleChainId ruleChainId) {
Validator.validateId(ruleChainId, "Incorrect rule chain id.");

6
dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java

@ -44,6 +44,7 @@ import org.thingsboard.server.dao.service.PaginatedRemover;
import org.thingsboard.server.dao.service.Validator;
import org.thingsboard.server.dao.service.validator.TenantDataValidator;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.dao.trendz.TrendzSettingsService;
import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
import org.thingsboard.server.dao.user.UserService;
@ -81,6 +82,8 @@ public class TenantServiceImpl extends AbstractCachedEntityService<TenantId, Ten
@Autowired
private QrCodeSettingService qrCodeSettingService;
@Autowired
private TrendzSettingsService trendzSettingsService;
@Autowired
private TenantDataValidator tenantValidator;
@Autowired
protected TbTransactionalCache<TenantId, Boolean> existsTenantCache;
@ -163,9 +166,10 @@ public class TenantServiceImpl extends AbstractCachedEntityService<TenantId, Ten
Validator.validateId(tenantId, id -> INCORRECT_TENANT_ID + id);
userService.deleteAllByTenantId(tenantId);
notificationSettingsService.deleteNotificationSettings(tenantId);
trendzSettingsService.deleteTrendzSettings(tenantId);
adminSettingsService.deleteAdminSettingsByTenantId(tenantId);
qrCodeSettingService.deleteByTenantId(tenantId);
notificationSettingsService.deleteNotificationSettings(tenantId);
tenantDao.removeById(tenantId, tenantId.getId());
publishEvictEvent(new TenantEvictEvent(tenantId, true));

69
dao/src/main/java/org/thingsboard/server/dao/trendz/DefaultTrendzSettingsService.java

@ -0,0 +1,69 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.trendz;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.CacheConstants;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.trendz.TrendzSettings;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import java.util.Optional;
@Service
@RequiredArgsConstructor
@Slf4j
public class DefaultTrendzSettingsService implements TrendzSettingsService {
private final AdminSettingsService adminSettingsService;
private static final String SETTINGS_KEY = "trendz";
@CacheEvict(cacheNames = CacheConstants.TRENDZ_SETTINGS_CACHE, key = "#tenantId")
@Override
public void saveTrendzSettings(TenantId tenantId, TrendzSettings settings) {
AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByTenantIdAndKey(tenantId, SETTINGS_KEY))
.orElseGet(() -> {
AdminSettings newAdminSettings = new AdminSettings();
newAdminSettings.setTenantId(tenantId);
newAdminSettings.setKey(SETTINGS_KEY);
return newAdminSettings;
});
adminSettings.setJsonValue(JacksonUtil.valueToTree(settings));
adminSettingsService.saveAdminSettings(tenantId, adminSettings);
}
@Cacheable(cacheNames = CacheConstants.TRENDZ_SETTINGS_CACHE, key = "#tenantId")
@Override
public TrendzSettings findTrendzSettings(TenantId tenantId) {
return Optional.ofNullable(adminSettingsService.findAdminSettingsByTenantIdAndKey(tenantId, SETTINGS_KEY))
.map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), TrendzSettings.class))
.orElseGet(TrendzSettings::new);
}
@CacheEvict(cacheNames = CacheConstants.TRENDZ_SETTINGS_CACHE, key = "#tenantId")
@Override
public void deleteTrendzSettings(TenantId tenantId) {
adminSettingsService.deleteAdminSettingsByTenantIdAndKey(tenantId, SETTINGS_KEY);
}
}

125
dao/src/test/java/org/thingsboard/server/dao/service/RuleChainServiceTest.java

@ -28,12 +28,14 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.rule.RuleChainService;
import java.io.IOException;
@ -43,7 +45,10 @@ import java.util.List;
import java.util.UUID;
import java.util.function.Function;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.thingsboard.server.common.data.relation.EntityRelation.USES_TYPE;
import static org.thingsboard.server.dao.rule.BaseRuleChainService.TB_RULE_CHAIN_INPUT_NODE;
/**
* Created by igor on 3/13/18.
@ -55,6 +60,8 @@ public class RuleChainServiceTest extends AbstractServiceTest {
EdgeService edgeService;
@Autowired
RuleChainService ruleChainService;
@Autowired
RelationService relationService;
private IdComparator<RuleChain> idComparator = new IdComparator<>();
private IdComparator<RuleNode> ruleNodeIdComparator = new IdComparator<>();
@ -270,7 +277,7 @@ public class RuleChainServiceTest extends AbstractServiceTest {
List<RuleNode> ruleNodes = savedRuleChainMetaData.getNodes();
int name3Index = -1;
for (int i=0;i<ruleNodes.size();i++) {
for (int i = 0; i < ruleNodes.size(); i++) {
if ("name3".equals(ruleNodes.get(i).getName())) {
name3Index = i;
break;
@ -354,6 +361,80 @@ public class RuleChainServiceTest extends AbstractServiceTest {
Assert.assertTrue(ruleChainById.isRoot());
}
@Test
public void testSaveRuleChainWithInputNode() {
RuleChain fromRuleChain = new RuleChain();
fromRuleChain.setName("From RuleChain");
fromRuleChain.setTenantId(tenantId);
fromRuleChain = ruleChainService.saveRuleChain(fromRuleChain);
RuleChainId fromRuleChainId = fromRuleChain.getId();
RuleChain toRuleChain1 = new RuleChain();
toRuleChain1.setName("To Rule Chain 1");
toRuleChain1.setTenantId(tenantId);
toRuleChain1 = ruleChainService.saveRuleChain(toRuleChain1);
RuleChainId toRuleChain1Id = toRuleChain1.getId();
RuleChainMetaData ruleChainMetaData = new RuleChainMetaData();
ruleChainMetaData.setRuleChainId(fromRuleChainId);
RuleNode toRuleChain1Node = new RuleNode();
toRuleChain1Node.setName("To Rule Chain 1");
toRuleChain1Node.setType(TB_RULE_CHAIN_INPUT_NODE);
toRuleChain1Node.setConfiguration(JacksonUtil.newObjectNode()
.put("ruleChainId", toRuleChain1Id.toString()));
RuleNode toRuleChain1Node2 = new RuleNode();
toRuleChain1Node2.setName("To Rule Chain 1");
toRuleChain1Node2.setType(TB_RULE_CHAIN_INPUT_NODE);
toRuleChain1Node2.setConfiguration(JacksonUtil.newObjectNode()
.put("ruleChainId", toRuleChain1Id.toString()));
List<RuleNode> ruleNodes = new ArrayList<>();
ruleNodes.add(toRuleChain1Node);
ruleNodes.add(toRuleChain1Node2);
ruleChainMetaData.setFirstNodeIndex(0);
ruleChainMetaData.setNodes(ruleNodes);
ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData, Function.identity());
List<EntityRelation> relations = relationService.findByFromAndType(tenantId, fromRuleChainId, USES_TYPE, RelationTypeGroup.COMMON);
assertThat(relations).singleElement().satisfies(relationToRuleChain1 -> {
assertThat(relationToRuleChain1.getFrom()).isEqualTo(fromRuleChainId);
assertThat(relationToRuleChain1.getTo()).isEqualTo(toRuleChain1Id);
});
RuleChain toRuleChain2 = new RuleChain();
toRuleChain2.setName("To Rule Chain 2");
toRuleChain2.setTenantId(tenantId);
toRuleChain2 = ruleChainService.saveRuleChain(toRuleChain2);
RuleChainId toRuleChain2Id = toRuleChain2.getId();
RuleNode toRuleChain2Node = new RuleNode();
toRuleChain2Node.setName("To Rule Chain 2");
toRuleChain2Node.setType(TB_RULE_CHAIN_INPUT_NODE);
toRuleChain2Node.setConfiguration(JacksonUtil.newObjectNode()
.put("ruleChainId", toRuleChain2Id.toString()));
List<RuleNode> newRuleNodes = new ArrayList<>();
newRuleNodes.add(toRuleChain2Node);
newRuleNodes.add(toRuleChain1Node);
ruleChainMetaData = ruleChainService.loadRuleChainMetaData(tenantId, ruleChainMetaData.getRuleChainId());
ruleChainMetaData.setNodes(newRuleNodes);
ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData, Function.identity());
List<EntityRelation> newRelations = relationService.findByFromAndType(tenantId, fromRuleChainId, USES_TYPE, RelationTypeGroup.COMMON);
assertThat(newRelations).hasSize(2);
assertThat(newRelations).anySatisfy(relationToRuleChain1 -> {
assertThat(relationToRuleChain1.getFrom()).isEqualTo(fromRuleChainId);
assertThat(relationToRuleChain1.getTo()).isEqualTo(toRuleChain1Id);
});
assertThat(newRelations).anySatisfy(relationToRuleChain2 -> {
assertThat(relationToRuleChain2.getFrom()).isEqualTo(fromRuleChainId);
assertThat(relationToRuleChain2.getTo()).isEqualTo(toRuleChain2Id);
});
}
private RuleChainId saveRuleChainAndSetAutoAssignToEdge(String name) {
RuleChain edgeRuleChain = new RuleChain();
edgeRuleChain.setTenantId(tenantId);
@ -395,9 +476,9 @@ public class RuleChainServiceTest extends AbstractServiceTest {
ruleChainMetaData.setFirstNodeIndex(0);
ruleChainMetaData.setNodes(ruleNodes);
ruleChainMetaData.addConnectionInfo(0,1,"success");
ruleChainMetaData.addConnectionInfo(0,2,"fail");
ruleChainMetaData.addConnectionInfo(1,2,"success");
ruleChainMetaData.addConnectionInfo(0, 1, "success");
ruleChainMetaData.addConnectionInfo(0, 2, "fail");
ruleChainMetaData.addConnectionInfo(1, 2, "success");
Assert.assertTrue(ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData, Function.identity()).isSuccess());
return ruleChainService.loadRuleChainMetaData(tenantId, ruleChainMetaData.getRuleChainId());
@ -434,10 +515,10 @@ public class RuleChainServiceTest extends AbstractServiceTest {
ruleChainMetaData.setFirstNodeIndex(0);
ruleChainMetaData.setNodes(ruleNodes);
ruleChainMetaData.addConnectionInfo(0,1,"success");
ruleChainMetaData.addConnectionInfo(0,2,"fail");
ruleChainMetaData.addConnectionInfo(1,2,"success");
ruleChainMetaData.addConnectionInfo(2,2,"success");
ruleChainMetaData.addConnectionInfo(0, 1, "success");
ruleChainMetaData.addConnectionInfo(0, 2, "fail");
ruleChainMetaData.addConnectionInfo(1, 2, "success");
ruleChainMetaData.addConnectionInfo(2, 2, "success");
return ruleChainMetaData;
}
@ -473,10 +554,10 @@ public class RuleChainServiceTest extends AbstractServiceTest {
ruleChainMetaData.setFirstNodeIndex(0);
ruleChainMetaData.setNodes(ruleNodes);
ruleChainMetaData.addConnectionInfo(0,1,"success");
ruleChainMetaData.addConnectionInfo(0,2,"fail");
ruleChainMetaData.addConnectionInfo(1,2,"success");
ruleChainMetaData.addConnectionInfo(2,0,"success");
ruleChainMetaData.addConnectionInfo(0, 1, "success");
ruleChainMetaData.addConnectionInfo(0, 2, "fail");
ruleChainMetaData.addConnectionInfo(1, 2, "success");
ruleChainMetaData.addConnectionInfo(2, 0, "success");
return ruleChainMetaData;
}
@ -582,16 +663,16 @@ public class RuleChainServiceTest extends AbstractServiceTest {
private RuleChain getRuleChain() {
String ruleChainStr = "{\n" +
" \"name\": \"Root Rule Chain\",\n" +
" \"type\": \"CORE\",\n" +
" \"firstRuleNodeId\": {\n" +
" \"entityType\": \"RULE_NODE\",\n" +
" \"id\": \"91ad0b00-e779-11ee-9cf0-15d8b6079fdb\"\n" +
" },\n" +
" \"debugMode\": false,\n" +
" \"configuration\": null,\n" +
" \"additionalInfo\": null\n" +
"}";
" \"name\": \"Root Rule Chain\",\n" +
" \"type\": \"CORE\",\n" +
" \"firstRuleNodeId\": {\n" +
" \"entityType\": \"RULE_NODE\",\n" +
" \"id\": \"91ad0b00-e779-11ee-9cf0-15d8b6079fdb\"\n" +
" },\n" +
" \"debugMode\": false,\n" +
" \"configuration\": null,\n" +
" \"additionalInfo\": null\n" +
"}";
return JacksonUtil.fromString(ruleChainStr, RuleChain.class);
}
}

3
dao/src/test/resources/application-test.properties

@ -108,6 +108,9 @@ cache.specs.qrCodeSettings.maxSize=10000
cache.specs.mobileSecretKey.timeToLiveInMinutes=1440
cache.specs.mobileSecretKey.maxSize=10000
cache.specs.trendzSettings.timeToLiveInMinutes=1440
cache.specs.trendzSettings.maxSize=10000
redis.connection.host=localhost
redis.connection.port=6379
redis.connection.db=0

6
edqs/src/main/resources/edqs.yml

@ -148,7 +148,11 @@ queue:
- key: max.poll.records
# Max poll records for edqs.state topic
value: "${TB_QUEUE_KAFKA_EDQS_STATE_MAX_POLL_RECORDS:512}"
# If you override any default Kafka topic name using environment variables, you must also specify the related consumer properties
# for the new topic in `consumer-properties-per-topic-inline`. Otherwise, the topic will not inherit its expected configuration (e.g., max.poll.records, timeouts, etc).
# Format: "topic1:key1=value1,key2=value2;topic2:key=value"
# Example: "tb_core_modified.notifications:max.poll.records=10;tb_edge_modified:max.poll.records=10,enable.auto.commit=true"
consumer-properties-per-topic-inline: "${TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE:}"
other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000"
other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside
# - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms

12
msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java

@ -55,8 +55,8 @@ public class ContainerTestSuite {
private static final String TB_JS_EXECUTOR_LOG_REGEXP = ".*template started.*";
private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(400);
private DockerComposeContainer<?> testContainer;
private ThingsBoardDbInstaller installTb;
private DockerComposeContainer<?> testContainer;
private ThingsBoardDbInstaller installTb;
private boolean isActive;
private static ContainerTestSuite containerTestSuite;
@ -194,7 +194,7 @@ public class ContainerTestSuite {
setActive(true);
} catch (Exception e) {
log.error("Failed to create test container", e);
fail("Failed to create test container");
fail("Failed to create test container", e);
}
}
@ -263,7 +263,7 @@ public class ContainerTestSuite {
log.info("Trying to delete temp dir {}", targetDir);
FileUtils.deleteDirectory(new File(targetDir));
} catch (IOException e) {
log.error("Can't delete temp directory " + targetDir, e);
log.error("Can't delete temp directory {}", targetDir, e);
}
}
@ -286,8 +286,8 @@ public class ContainerTestSuite {
FileUtils.writeStringToFile(file, outputContent, StandardCharsets.UTF_8);
assertThat(FileUtils.readFileToString(file, StandardCharsets.UTF_8), is(outputContent));
} catch (IOException e) {
log.error("failed to update file " + sourceFilename, e);
fail("failed to update file");
log.error("failed to update file {}", sourceFilename, e);
fail("failed to update file", e);
}
}

15
msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java

@ -42,7 +42,6 @@ import org.thingsboard.mqtt.MqttClient;
import org.thingsboard.mqtt.MqttClientCallback;
import org.thingsboard.mqtt.MqttClientConfig;
import org.thingsboard.mqtt.MqttHandler;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.DeviceProfileProvisionType;
@ -82,7 +81,6 @@ import java.util.concurrent.TimeoutException;
import static org.assertj.core.api.Assertions.assertThat;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.fail;
import static org.thingsboard.server.common.data.DataConstants.DEVICE;
import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE;
import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultDevicePrototype;
@ -301,7 +299,7 @@ public class MqttClientTest extends AbstractContainerTest {
assertThat(Objects.requireNonNull(requestFromServer).getMessage()).isEqualTo("{\"method\":\"getValue\",\"params\":true}");
Integer requestId = Integer.valueOf(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length()));
int requestId = Integer.parseInt(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length()));
JsonObject clientResponse = new JsonObject();
clientResponse.addProperty("response", "someResponse");
// Send a response to the server's RPC request
@ -340,7 +338,7 @@ public class MqttClientTest extends AbstractContainerTest {
assertThat(Objects.requireNonNull(requestFromServer).getMessage()).isEqualTo("{\"method\":\"getValue\",\"params\":true}");
Integer requestId = Integer.valueOf(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length()));
int requestId = Integer.parseInt(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length()));
JsonObject clientResponse = new JsonObject();
clientResponse.addProperty("response", "someResponse");
// Send a response to the server's RPC request
@ -520,13 +518,13 @@ public class MqttClientTest extends AbstractContainerTest {
mqttClient.on("/provision/response", listener, MqttQoS.AT_LEAST_ONCE).get(3 * timeoutMultiplier, TimeUnit.SECONDS);
TimeUnit.SECONDS.sleep(2 * timeoutMultiplier);
assertThat(subAckResult[0]).isNotNull();
assertThat(MqttReasonCodes.SubAck.GRANTED_QOS_1.equals(subAckResult[0]));
assertThat(MqttReasonCodes.SubAck.GRANTED_QOS_1).isEqualTo(subAckResult[0]);
subAckResult[0] = null;
mqttClient.on("v1/devices/me/attributes", listener, MqttQoS.AT_LEAST_ONCE).get(3 * timeoutMultiplier, TimeUnit.SECONDS);
TimeUnit.SECONDS.sleep(2 * timeoutMultiplier);
assertThat(subAckResult[0]).isNotNull();
assertThat(MqttReasonCodes.SubAck.TOPIC_FILTER_INVALID.equals(subAckResult[0]));
assertThat(MqttReasonCodes.SubAck.TOPIC_FILTER_INVALID).isEqualTo(subAckResult[0]);
testRestClient.deleteDeviceIfExists(device.getId());
updateDeviceProfileWithProvisioningStrategy(deviceProfile, DeviceProfileProvisionType.DISABLED);
@ -596,7 +594,7 @@ public class MqttClientTest extends AbstractContainerTest {
.await()
.alias("Check device disconnect.")
.atMost(TIMEOUT*timeoutMultiplier, TimeUnit.SECONDS)
.until(() -> returnCodeByteValue.size() > 0);
.until(() -> !returnCodeByteValue.isEmpty());
assertThat(returnCodeByteValueSecondClient).isEmpty();
assertThat(returnCodeByteValue).isNotEmpty();
@ -663,7 +661,7 @@ public class MqttClientTest extends AbstractContainerTest {
.stream()
.filter(RuleChain::isRoot)
.findFirst();
if (!defaultRuleChain.isPresent()) {
if (defaultRuleChain.isEmpty()) {
fail("Root rule chain wasn't found");
}
return defaultRuleChain.get().getId();
@ -717,6 +715,7 @@ public class MqttClientTest extends AbstractContainerTest {
clientConfig.setClientId("MQTT client from test");
clientConfig.setUsername(username);
clientConfig.setProtocolVersion(mqttVersion);
clientConfig.setRetransmissionConfig(new MqttClientConfig.RetransmissionConfig(3, 5000L, 0.15d)); // same as defaults in thingsboard.yml as of time of this writing
MqttClient mqttClient = MqttClient.create(clientConfig, listener, handlerExecutor);
if (connect) {
mqttClient.connect(TRANSPORT_HOST, TRANSPORT_PORT).get();

14
msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java

@ -39,7 +39,6 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.mqtt.MqttClient;
import org.thingsboard.mqtt.MqttClientConfig;
import org.thingsboard.mqtt.MqttHandler;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.id.DeviceId;
@ -65,7 +64,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat;
import static org.thingsboard.server.common.data.DataConstants.DEVICE;
import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE;
import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultGatewayPrototype;
@ -76,7 +74,6 @@ public class MqttGatewayClientTest extends AbstractContainerTest {
private MqttClient mqttClient;
private Device createdDevice;
private MqttMessageListener listener;
private JsonParser jsonParser = new JsonParser();
AbstractListeningExecutor handlerExecutor;
@ -100,7 +97,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest {
}
@AfterMethod
public void removeGateway() {
public void removeGateway() {
testRestClient.deleteDeviceIfExists(this.gatewayDevice.getId());
testRestClient.deleteDeviceIfExists(this.createdDevice.getId());
this.listener = null;
@ -197,7 +194,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest {
mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(requestData.toString().getBytes())).get();
event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS);
JsonObject responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject();
JsonObject responseData = JsonParser.parseString(Objects.requireNonNull(event).getMessage()).getAsJsonObject();
assertThat(responseData.has("value")).isTrue();
assertThat(responseData.get("value").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString());
@ -213,7 +210,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest {
mqttClient.on("v1/gateway/attributes/response", listener, MqttQoS.AT_LEAST_ONCE).get();
mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(requestData.toString().getBytes())).get();
event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS);
responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject();
responseData = JsonParser.parseString(Objects.requireNonNull(event).getMessage()).getAsJsonObject();
assertThat(responseData.has("values")).isTrue();
assertThat(responseData.get("values").getAsJsonObject().get("attr1").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString());
@ -231,7 +228,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest {
mqttClient.on("v1/gateway/attributes/response", listener, MqttQoS.AT_LEAST_ONCE).get();
mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(requestData.toString().getBytes())).get();
event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS);
responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject();
responseData = JsonParser.parseString(Objects.requireNonNull(event).getMessage()).getAsJsonObject();
assertThat(responseData.has("values")).isTrue();
assertThat(responseData.get("values").getAsJsonObject().get("attr1").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString());
@ -390,7 +387,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest {
mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(gatewayAttributesRequest.toString().getBytes())).get();
MqttEvent clientAttributeEvent = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS);
assertThat(clientAttributeEvent).isNotNull();
JsonObject responseMessage = new JsonParser().parse(Objects.requireNonNull(clientAttributeEvent).getMessage()).getAsJsonObject();
JsonObject responseMessage = JsonParser.parseString(Objects.requireNonNull(clientAttributeEvent).getMessage()).getAsJsonObject();
assertThat(responseMessage.get("id").getAsInt()).isEqualTo(messageId);
assertThat(responseMessage.get("device").getAsString()).isEqualTo(createdDevice.getName());
@ -427,6 +424,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest {
clientConfig.setOwnerId(getOwnerId());
clientConfig.setClientId("MQTT client from test");
clientConfig.setUsername(deviceCredentials.getCredentialsId());
clientConfig.setRetransmissionConfig(new MqttClientConfig.RetransmissionConfig(3, 5000L, 0.15d)); // same as defaults in thingsboard.yml as of time of this writing
MqttClient mqttClient = MqttClient.create(clientConfig, listener, handlerExecutor);
mqttClient.connect("localhost", 1883).get();
return mqttClient;

5
msa/vc-executor/src/main/resources/tb-vc-executor.yml

@ -124,6 +124,11 @@ queue:
# tb_rule_engine.sq:
# - key: max.poll.records
# value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}"
# If you override any default Kafka topic name using environment variables, you must also specify the related consumer properties
# for the new topic in `consumer-properties-per-topic-inline`. Otherwise, the topic will not inherit its expected configuration (e.g., max.poll.records, timeouts, etc).
# Format: "topic1:key1=value1,key2=value2;topic2:key=value"
# Example: "tb_core_modified.notifications:max.poll.records=10;tb_edge_modified:max.poll.records=10,enable.auto.commit=true"
consumer-properties-per-topic-inline: "${TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE:}"
other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000"
other: # DEPRECATED. In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside
# - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms

20
netty-mqtt/pom.xml

@ -87,6 +87,26 @@
<artifactId>awaitility</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>software.xdev</groupId>
<artifactId>testcontainers-junit4-mock</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>hivemq</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>

24
netty-mqtt/src/main/java/org/thingsboard/mqtt/MaxRetransmissionsReachedException.java

@ -0,0 +1,24 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.mqtt;
public class MaxRetransmissionsReachedException extends RuntimeException {
public MaxRetransmissionsReachedException(String message) {
super(message);
}
}

30
netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttChannelHandler.java

@ -57,7 +57,7 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler<MqttMessage>
}
@Override
protected void channelRead0(ChannelHandlerContext ctx, MqttMessage msg) throws Exception {
protected void channelRead0(ChannelHandlerContext ctx, MqttMessage msg) {
if (msg.decoderResult().isSuccess()) {
switch (msg.fixedHeader().messageType()) {
case CONNACK:
@ -120,6 +120,7 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler<MqttMessage>
this.client.getClientConfig().getUsername(),
this.client.getClientConfig().getPassword() != null ? this.client.getClientConfig().getPassword().getBytes(CharsetUtil.UTF_8) : null
);
log.debug("{} Sending CONNECT", client.getClientConfig().getOwnerId());
ctx.channel().writeAndFlush(new MqttConnectMessage(fixedHeader, variableHeader, payload));
}
@ -173,6 +174,7 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler<MqttMessage>
}
private void handleConack(Channel channel, MqttConnAckMessage message) {
log.debug("{} Handling CONNACK", client.getClientConfig().getOwnerId());
switch (message.variableHeader().connectReturnCode()) {
case CONNECTION_ACCEPTED:
this.connectFuture.setSuccess(new MqttConnectResult(true, MqttConnectReturnCode.CONNECTION_ACCEPTED, channel.closeFuture()));
@ -219,9 +221,9 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler<MqttMessage>
}
pendingSubscription.onSubackReceived();
for (MqttPendingSubscription.MqttPendingHandler handler : pendingSubscription.getHandlers()) {
MqttSubscription subscription = new MqttSubscription(pendingSubscription.getTopic(), handler.getHandler(), handler.isOnce());
MqttSubscription subscription = new MqttSubscription(pendingSubscription.getTopic(), handler.handler(), handler.once());
this.client.getSubscriptions().put(pendingSubscription.getTopic(), subscription);
this.client.getHandlerToSubscription().put(handler.getHandler(), subscription);
this.client.getHandlerToSubscription().put(handler.handler(), subscription);
}
this.client.getPendingSubscribeTopics().remove(pendingSubscription.getTopic());
@ -282,17 +284,16 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler<MqttMessage>
}
private void handlePuback(MqttPubAckMessage message) {
MqttPendingPublish pendingPublish = this.client.getPendingPublishes().get(message.variableHeader().messageId());
if (pendingPublish == null) {
return;
}
pendingPublish.getFuture().setSuccess(null);
pendingPublish.onPubackReceived();
this.client.getPendingPublishes().remove(message.variableHeader().messageId());
pendingPublish.getPayload().release();
if (this.client.getCallback() != null) {
this.client.getCallback().onPubAck(message);
}
log.trace("{} Handling PUBACK", client.getClientConfig().getOwnerId());
client.getPendingPublishes().computeIfPresent(message.variableHeader().messageId(), (__, pendingPublish) -> {
pendingPublish.getFuture().setSuccess(null);
pendingPublish.onPubackReceived();
pendingPublish.getPayload().release();
if (client.getCallback() != null) {
client.getCallback().onPubAck(message);
}
return null;
});
}
private void handlePubrec(Channel channel, MqttMessage message) {
@ -335,6 +336,7 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler<MqttMessage>
}
private void handleDisconnect(MqttMessage message) {
log.debug("{} Handling DISCONNECT", client.getClientConfig().getOwnerId());
if (this.client.getCallback() != null) {
this.client.getCallback().onDisconnect(message);
}

2
netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClient.java

@ -184,7 +184,7 @@ public interface MqttClient {
* @param config The config object to use while looking for settings
* @param defaultHandler The handler for incoming messages that do not match any topic subscriptions
*/
static MqttClient create(MqttClientConfig config, MqttHandler defaultHandler, ListeningExecutor handlerExecutor){
static MqttClient create(MqttClientConfig config, MqttHandler defaultHandler, ListeningExecutor handlerExecutor) {
return new MqttClientImpl(config, defaultHandler, handlerExecutor);
}

20
netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientConfig.java

@ -47,6 +47,26 @@ public final class MqttClientConfig {
private long reconnectDelay = 1L;
private int maxBytesInMessage = 8092;
@Getter
@Setter
private RetransmissionConfig retransmissionConfig;
public record RetransmissionConfig(int maxAttempts, long initialDelayMillis, double jitterFactor) {
public RetransmissionConfig {
if (maxAttempts < 0) {
throw new IllegalArgumentException("Max retransmission attempts (maxAttempts) must be zero or greater, but was " + maxAttempts);
}
if (initialDelayMillis < 0) {
throw new IllegalArgumentException("Initial retransmission delay (initialDelayMillis) must be zero or greater, but was " + initialDelayMillis);
}
if (jitterFactor < 0) {
throw new IllegalArgumentException("Jitter factor (jitterFactor) must be zero or greater, but was " + jitterFactor);
}
}
}
public MqttClientConfig() {
this(null);
}

82
netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java

@ -17,6 +17,7 @@ package org.thingsboard.mqtt;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import io.netty.bootstrap.Bootstrap;
import io.netty.buffer.ByteBuf;
import io.netty.channel.Channel;
@ -384,8 +385,33 @@ final class MqttClientImpl implements MqttClient {
MqttFixedHeader fixedHeader = new MqttFixedHeader(MqttMessageType.PUBLISH, false, qos, retain, 0);
MqttPublishVariableHeader variableHeader = new MqttPublishVariableHeader(topic, getNewMessageId().messageId());
MqttPublishMessage message = new MqttPublishMessage(fixedHeader, variableHeader, payload);
MqttPendingPublish pendingPublish = new MqttPendingPublish(variableHeader.packetId(), future,
payload.retain(), message, qos, () -> !pendingPublishes.containsKey(variableHeader.packetId()));
final var pendingPublish = MqttPendingPublish.builder()
.messageId(variableHeader.packetId())
.future(future)
.payload(payload.retain())
.message(message)
.qos(qos)
.ownerId(clientConfig.getOwnerId())
.retransmissionConfig(clientConfig.getRetransmissionConfig())
.pendingOperation(new PendingOperation() {
@Override
public boolean isCancelled() {
return !pendingPublishes.containsKey(variableHeader.packetId());
}
@Override
public void onMaxRetransmissionAttemptsReached() {
pendingPublishes.computeIfPresent(variableHeader.packetId(), (__, pendingPublish) -> {
var message = "Unable to deliver publish message due to max retransmission attempts (%s) being reached for client '%s' on topic '%s' (message ID: %d)"
.formatted(clientConfig.getRetransmissionConfig().maxAttempts(), clientConfig.getClientId(), topic, variableHeader.packetId());
pendingPublish.getFuture().tryFailure(new MaxRetransmissionsReachedException(message));
pendingPublish.getPayload().release();
return null;
});
}
}).build();
this.pendingPublishes.put(pendingPublish.getMessageId(), pendingPublish);
ChannelFuture channelFuture = this.sendAndFlushPacket(message);
@ -499,9 +525,30 @@ final class MqttClientImpl implements MqttClient {
MqttSubscribePayload payload = new MqttSubscribePayload(Collections.singletonList(subscription));
MqttSubscribeMessage message = new MqttSubscribeMessage(fixedHeader, variableHeader, payload);
final MqttPendingSubscription pendingSubscription = new MqttPendingSubscription(future, topic, message,
() -> !pendingSubscriptions.containsKey(variableHeader.messageId()));
pendingSubscription.addHandler(handler, once);
final var pendingSubscription = MqttPendingSubscription.builder()
.future(future)
.topic(topic)
.handlers(Sets.newHashSet(new MqttPendingSubscription.MqttPendingHandler(handler, once)))
.subscribeMessage(message)
.ownerId(clientConfig.getOwnerId())
.retransmissionConfig(clientConfig.getRetransmissionConfig())
.pendingOperation(new PendingOperation() {
@Override
public boolean isCancelled() {
return !pendingSubscriptions.containsKey(variableHeader.messageId());
}
@Override
public void onMaxRetransmissionAttemptsReached() {
pendingSubscriptions.computeIfPresent(variableHeader.messageId(), (__, pendingSubscription) -> {
var message = "Unable to deliver subscribe message due to max retransmission attempts (%s) being reached for client '%s' on topic '%s' (message ID: %d)"
.formatted(clientConfig.getRetransmissionConfig().maxAttempts(), clientConfig.getClientId(), topic, variableHeader.messageId());
pendingSubscription.getFuture().tryFailure(new MaxRetransmissionsReachedException(message));
return null;
});
}
}).build();
this.pendingSubscriptions.put(variableHeader.messageId(), pendingSubscription);
this.pendingSubscribeTopics.add(topic);
pendingSubscription.setSent(this.sendAndFlushPacket(message) != null); //If not sent, we will send it when the connection is opened
@ -518,8 +565,29 @@ final class MqttClientImpl implements MqttClient {
MqttUnsubscribePayload payload = new MqttUnsubscribePayload(Collections.singletonList(topic));
MqttUnsubscribeMessage message = new MqttUnsubscribeMessage(fixedHeader, variableHeader, payload);
MqttPendingUnsubscription pendingUnsubscription = new MqttPendingUnsubscription(promise, topic, message,
() -> !pendingServerUnsubscribes.containsKey(variableHeader.messageId()));
final var pendingUnsubscription = MqttPendingUnsubscription.builder()
.future(promise)
.topic(topic)
.unsubscribeMessage(message)
.ownerId(clientConfig.getOwnerId())
.retransmissionConfig(clientConfig.getRetransmissionConfig())
.pendingOperation(new PendingOperation() {
@Override
public boolean isCancelled() {
return !pendingServerUnsubscribes.containsKey(variableHeader.messageId());
}
@Override
public void onMaxRetransmissionAttemptsReached() {
pendingServerUnsubscribes.computeIfPresent(variableHeader.messageId(), (__, pendingUnsubscription) -> {
var message = "Unable to deliver unsubscribe message due to max retransmission attempts (%s) being reached for client '%s' on topic '%s' (message ID: %d)"
.formatted(clientConfig.getRetransmissionConfig().maxAttempts(), clientConfig.getClientId(), topic, variableHeader.messageId());
pendingUnsubscription.getFuture().tryFailure(new MaxRetransmissionsReachedException(message));
return null;
});
}
}).build();
this.pendingServerUnsubscribes.put(variableHeader.messageId(), pendingUnsubscription);
pendingUnsubscription.startRetransmissionTimer(this.eventLoop.next(), this::sendAndFlushPacket);

3
netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttConnectResult.java

@ -17,7 +17,9 @@ package org.thingsboard.mqtt;
import io.netty.channel.ChannelFuture;
import io.netty.handler.codec.mqtt.MqttConnectReturnCode;
import lombok.ToString;
@ToString
@SuppressWarnings({"WeakerAccess", "unused"})
public final class MqttConnectResult {
@ -42,4 +44,5 @@ public final class MqttConnectResult {
public ChannelFuture getCloseFuture() {
return closeFuture;
}
}

134
netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingPublish.java

@ -21,9 +21,13 @@ import io.netty.handler.codec.mqtt.MqttMessage;
import io.netty.handler.codec.mqtt.MqttPublishMessage;
import io.netty.handler.codec.mqtt.MqttQoS;
import io.netty.util.concurrent.Promise;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.Setter;
import java.util.function.Consumer;
@Getter(AccessLevel.PACKAGE)
final class MqttPendingPublish {
private final int messageId;
@ -32,80 +36,126 @@ final class MqttPendingPublish {
private final MqttPublishMessage message;
private final MqttQoS qos;
@Getter(AccessLevel.NONE)
private final RetransmissionHandler<MqttPublishMessage> publishRetransmissionHandler;
@Getter(AccessLevel.NONE)
private final RetransmissionHandler<MqttMessage> pubrelRetransmissionHandler;
@Setter(AccessLevel.PACKAGE)
private boolean sent = false;
MqttPendingPublish(int messageId, Promise<Void> future, ByteBuf payload, MqttPublishMessage message, MqttQoS qos, PendingOperation operation) {
private MqttPendingPublish(
int messageId,
Promise<Void> future,
ByteBuf payload,
MqttPublishMessage message,
MqttQoS qos,
String ownerId,
MqttClientConfig.RetransmissionConfig retransmissionConfig,
PendingOperation pendingOperation
) {
this.messageId = messageId;
this.future = future;
this.payload = payload;
this.message = message;
this.qos = qos;
this.publishRetransmissionHandler = new RetransmissionHandler<>(operation);
this.publishRetransmissionHandler.setOriginalMessage(message);
this.pubrelRetransmissionHandler = new RetransmissionHandler<>(operation);
}
int getMessageId() {
return messageId;
}
Promise<Void> getFuture() {
return future;
}
ByteBuf getPayload() {
return payload;
}
boolean isSent() {
return sent;
}
void setSent(boolean sent) {
this.sent = sent;
}
MqttPublishMessage getMessage() {
return message;
}
MqttQoS getQos() {
return qos;
publishRetransmissionHandler = new RetransmissionHandler<>(retransmissionConfig, pendingOperation, ownerId);
publishRetransmissionHandler.setOriginalMessage(message);
pubrelRetransmissionHandler = new RetransmissionHandler<>(retransmissionConfig, pendingOperation, ownerId);
}
void startPublishRetransmissionTimer(EventLoop eventLoop, Consumer<Object> sendPacket) {
this.publishRetransmissionHandler.setHandle(((fixedHeader, originalMessage) ->
sendPacket.accept(new MqttPublishMessage(fixedHeader, originalMessage.variableHeader(), this.payload.retain()))));
this.publishRetransmissionHandler.start(eventLoop);
publishRetransmissionHandler.setHandler(((fixedHeader, originalMessage) ->
sendPacket.accept(new MqttPublishMessage(fixedHeader, originalMessage.variableHeader(), payload.retain()))));
publishRetransmissionHandler.start(eventLoop);
}
void onPubackReceived() {
this.publishRetransmissionHandler.stop();
publishRetransmissionHandler.stop();
}
void setPubrelMessage(MqttMessage pubrelMessage) {
this.pubrelRetransmissionHandler.setOriginalMessage(pubrelMessage);
pubrelRetransmissionHandler.setOriginalMessage(pubrelMessage);
}
void startPubrelRetransmissionTimer(EventLoop eventLoop, Consumer<Object> sendPacket) {
this.pubrelRetransmissionHandler.setHandle((fixedHeader, originalMessage) ->
pubrelRetransmissionHandler.setHandler((fixedHeader, originalMessage) ->
sendPacket.accept(new MqttMessage(fixedHeader, originalMessage.variableHeader())));
this.pubrelRetransmissionHandler.start(eventLoop);
pubrelRetransmissionHandler.start(eventLoop);
}
void onPubcompReceived() {
this.pubrelRetransmissionHandler.stop();
pubrelRetransmissionHandler.stop();
}
void onChannelClosed() {
this.publishRetransmissionHandler.stop();
this.pubrelRetransmissionHandler.stop();
publishRetransmissionHandler.stop();
pubrelRetransmissionHandler.stop();
if (payload != null) {
payload.release();
}
}
static Builder builder() {
return new Builder();
}
static class Builder {
private int messageId;
private Promise<Void> future;
private ByteBuf payload;
private MqttPublishMessage message;
private MqttQoS qos;
private String ownerId;
private MqttClientConfig.RetransmissionConfig retransmissionConfig;
private PendingOperation pendingOperation;
Builder messageId(int messageId) {
this.messageId = messageId;
return this;
}
Builder future(Promise<Void> future) {
this.future = future;
return this;
}
Builder payload(ByteBuf payload) {
this.payload = payload;
return this;
}
Builder message(MqttPublishMessage message) {
this.message = message;
return this;
}
Builder qos(MqttQoS qos) {
this.qos = qos;
return this;
}
Builder ownerId(String ownerId) {
this.ownerId = ownerId;
return this;
}
Builder retransmissionConfig(MqttClientConfig.RetransmissionConfig retransmissionConfig) {
this.retransmissionConfig = retransmissionConfig;
return this;
}
Builder pendingOperation(PendingOperation pendingOperation) {
this.pendingOperation = pendingOperation;
return this;
}
MqttPendingPublish build() {
return new MqttPendingPublish(messageId, future, payload, message, qos, ownerId, retransmissionConfig, pendingOperation);
}
}
}

119
netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingSubscription.java

@ -18,90 +18,123 @@ package org.thingsboard.mqtt;
import io.netty.channel.EventLoop;
import io.netty.handler.codec.mqtt.MqttSubscribeMessage;
import io.netty.util.concurrent.Promise;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.Setter;
import java.util.HashSet;
import java.util.Set;
import java.util.function.Consumer;
import static java.util.Objects.requireNonNullElseGet;
@Getter(AccessLevel.PACKAGE)
final class MqttPendingSubscription {
private final Promise<Void> future;
private final String topic;
private final Set<MqttPendingHandler> handlers = new HashSet<>();
private final Set<MqttPendingHandler> handlers;
private final MqttSubscribeMessage subscribeMessage;
@Getter(AccessLevel.NONE)
private final RetransmissionHandler<MqttSubscribeMessage> retransmissionHandler;
@Setter(AccessLevel.PACKAGE)
private boolean sent = false;
MqttPendingSubscription(Promise<Void> future, String topic, MqttSubscribeMessage message, PendingOperation operation) {
private MqttPendingSubscription(
Promise<Void> future,
String topic,
Set<MqttPendingHandler> handlers,
MqttSubscribeMessage subscribeMessage,
String ownerId,
MqttClientConfig.RetransmissionConfig retransmissionConfig,
PendingOperation operation
) {
this.future = future;
this.topic = topic;
this.subscribeMessage = message;
this.handlers = requireNonNullElseGet(handlers, HashSet::new);
this.subscribeMessage = subscribeMessage;
this.retransmissionHandler = new RetransmissionHandler<>(operation);
this.retransmissionHandler.setOriginalMessage(message);
retransmissionHandler = new RetransmissionHandler<>(retransmissionConfig, operation, ownerId);
retransmissionHandler.setOriginalMessage(subscribeMessage);
}
Promise<Void> getFuture() {
return future;
}
record MqttPendingHandler(MqttHandler handler, boolean once) {}
String getTopic() {
return topic;
void addHandler(MqttHandler handler, boolean once) {
handlers.add(new MqttPendingHandler(handler, once));
}
boolean isSent() {
return sent;
void startRetransmitTimer(EventLoop eventLoop, Consumer<Object> sendPacket) {
if (sent) { // If the packet is sent, we can start the retransmission timer
retransmissionHandler.setHandler((fixedHeader, originalMessage) ->
sendPacket.accept(new MqttSubscribeMessage(fixedHeader, originalMessage.variableHeader(), originalMessage.payload())));
retransmissionHandler.start(eventLoop);
}
}
void setSent(boolean sent) {
this.sent = sent;
void onSubackReceived() {
retransmissionHandler.stop();
}
MqttSubscribeMessage getSubscribeMessage() {
return subscribeMessage;
void onChannelClosed() {
retransmissionHandler.stop();
}
void addHandler(MqttHandler handler, boolean once) {
this.handlers.add(new MqttPendingHandler(handler, once));
static Builder builder() {
return new Builder();
}
Set<MqttPendingHandler> getHandlers() {
return handlers;
}
static class Builder {
void startRetransmitTimer(EventLoop eventLoop, Consumer<Object> sendPacket) {
if (this.sent) { //If the packet is sent, we can start the retransmit timer
this.retransmissionHandler.setHandle((fixedHeader, originalMessage) ->
sendPacket.accept(new MqttSubscribeMessage(fixedHeader, originalMessage.variableHeader(), originalMessage.payload())));
this.retransmissionHandler.start(eventLoop);
private Promise<Void> future;
private String topic;
private Set<MqttPendingHandler> handlers;
private MqttSubscribeMessage subscribeMessage;
private String ownerId;
private PendingOperation pendingOperation;
private MqttClientConfig.RetransmissionConfig retransmissionConfig;
Builder future(Promise<Void> future) {
this.future = future;
return this;
}
}
void onSubackReceived() {
this.retransmissionHandler.stop();
}
Builder topic(String topic) {
this.topic = topic;
return this;
}
final class MqttPendingHandler {
private final MqttHandler handler;
private final boolean once;
Builder handlers(Set<MqttPendingHandler> handlers) {
this.handlers = handlers;
return this;
}
MqttPendingHandler(MqttHandler handler, boolean once) {
this.handler = handler;
this.once = once;
Builder subscribeMessage(MqttSubscribeMessage subscribeMessage) {
this.subscribeMessage = subscribeMessage;
return this;
}
MqttHandler getHandler() {
return handler;
Builder ownerId(String ownerId) {
this.ownerId = ownerId;
return this;
}
boolean isOnce() {
return once;
Builder retransmissionConfig(MqttClientConfig.RetransmissionConfig retransmissionConfig) {
this.retransmissionConfig = retransmissionConfig;
return this;
}
Builder pendingOperation(PendingOperation pendingOperation) {
this.pendingOperation = pendingOperation;
return this;
}
MqttPendingSubscription build() {
return new MqttPendingSubscription(future, topic, handlers, subscribeMessage, ownerId, retransmissionConfig, pendingOperation);
}
}
void onChannelClosed() {
this.retransmissionHandler.stop();
}
}

85
netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingUnsubscription.java

@ -18,43 +18,96 @@ package org.thingsboard.mqtt;
import io.netty.channel.EventLoop;
import io.netty.handler.codec.mqtt.MqttUnsubscribeMessage;
import io.netty.util.concurrent.Promise;
import lombok.AccessLevel;
import lombok.Getter;
import java.util.function.Consumer;
final class MqttPendingUnsubscription{
@Getter(AccessLevel.PACKAGE)
final class MqttPendingUnsubscription {
private final Promise<Void> future;
private final String topic;
@Getter(AccessLevel.NONE)
private final RetransmissionHandler<MqttUnsubscribeMessage> retransmissionHandler;
MqttPendingUnsubscription(Promise<Void> future, String topic, MqttUnsubscribeMessage unsubscribeMessage, PendingOperation operation) {
private MqttPendingUnsubscription(
Promise<Void> future,
String topic,
MqttUnsubscribeMessage unsubscribeMessage,
String ownerId,
MqttClientConfig.RetransmissionConfig retransmissionConfig,
PendingOperation operation
) {
this.future = future;
this.topic = topic;
this.retransmissionHandler = new RetransmissionHandler<>(operation);
this.retransmissionHandler.setOriginalMessage(unsubscribeMessage);
retransmissionHandler = new RetransmissionHandler<>(retransmissionConfig, operation, ownerId);
retransmissionHandler.setOriginalMessage(unsubscribeMessage);
}
Promise<Void> getFuture() {
return future;
void startRetransmissionTimer(EventLoop eventLoop, Consumer<Object> sendPacket) {
retransmissionHandler.setHandler((fixedHeader, originalMessage) ->
sendPacket.accept(new MqttUnsubscribeMessage(fixedHeader, originalMessage.variableHeader(), originalMessage.payload())));
retransmissionHandler.start(eventLoop);
}
String getTopic() {
return topic;
void onUnsubackReceived() {
retransmissionHandler.stop();
}
void startRetransmissionTimer(EventLoop eventLoop, Consumer<Object> sendPacket) {
this.retransmissionHandler.setHandle((fixedHeader, originalMessage) ->
sendPacket.accept(new MqttUnsubscribeMessage(fixedHeader, originalMessage.variableHeader(), originalMessage.payload())));
this.retransmissionHandler.start(eventLoop);
void onChannelClosed() {
retransmissionHandler.stop();
}
void onUnsubackReceived(){
this.retransmissionHandler.stop();
static Builder builder() {
return new Builder();
}
void onChannelClosed(){
this.retransmissionHandler.stop();
static class Builder {
private Promise<Void> future;
private String topic;
private MqttUnsubscribeMessage unsubscribeMessage;
private String ownerId;
private PendingOperation pendingOperation;
private MqttClientConfig.RetransmissionConfig retransmissionConfig;
Builder future(Promise<Void> future) {
this.future = future;
return this;
}
Builder topic(String topic) {
this.topic = topic;
return this;
}
Builder unsubscribeMessage(MqttUnsubscribeMessage unsubscribeMessage) {
this.unsubscribeMessage = unsubscribeMessage;
return this;
}
Builder ownerId(String ownerId) {
this.ownerId = ownerId;
return this;
}
Builder retransmissionConfig(MqttClientConfig.RetransmissionConfig retransmissionConfig) {
this.retransmissionConfig = retransmissionConfig;
return this;
}
Builder pendingOperation(PendingOperation pendingOperation) {
this.pendingOperation = pendingOperation;
return this;
}
MqttPendingUnsubscription build() {
return new MqttPendingUnsubscription(future, topic, unsubscribeMessage, ownerId, retransmissionConfig, pendingOperation);
}
}
}

17
netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPingHandler.java

@ -42,12 +42,11 @@ final class MqttPingHandler extends ChannelInboundHandlerAdapter {
}
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
if (!(msg instanceof MqttMessage)) {
public void channelRead(ChannelHandlerContext ctx, Object msg) {
if (!(msg instanceof MqttMessage message)) {
ctx.fireChannelRead(msg);
return;
}
MqttMessage message = (MqttMessage) msg;
if (message.fixedHeader().messageType() == MqttMessageType.PINGREQ) {
this.handlePingReq(ctx.channel());
} else if (message.fixedHeader().messageType() == MqttMessageType.PINGRESP) {
@ -61,28 +60,29 @@ final class MqttPingHandler extends ChannelInboundHandlerAdapter {
public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
super.userEventTriggered(ctx, evt);
if (evt instanceof IdleStateEvent) {
IdleStateEvent event = (IdleStateEvent) evt;
if (evt instanceof IdleStateEvent event) {
switch (event.state()) {
case READER_IDLE:
log.debug("[{}] No reads were performed for specified period for channel {}", event.state(), ctx.channel().id());
this.sendPingReq(ctx.channel());
this.sendPingReq(ctx.channel(), event);
break;
case WRITER_IDLE:
log.debug("[{}] No writes were performed for specified period for channel {}", event.state(), ctx.channel().id());
this.sendPingReq(ctx.channel());
this.sendPingReq(ctx.channel(), event);
break;
}
}
}
private void sendPingReq(Channel channel) {
private void sendPingReq(Channel channel, IdleStateEvent idleEvent) {
log.trace("[{}] Sending ping request", channel.id());
MqttFixedHeader fixedHeader = new MqttFixedHeader(MqttMessageType.PINGREQ, false, MqttQoS.AT_MOST_ONCE, false, 0);
channel.writeAndFlush(new MqttMessage(fixedHeader));
if (this.pingRespTimeout == null) {
log.trace("[{}] Scheduling disconnect due to {}", channel.id(), idleEvent);
this.pingRespTimeout = channel.eventLoop().schedule(() -> {
log.trace("[{}] Sending disconnect due to {}", channel.id(), idleEvent);
MqttFixedHeader fixedHeader2 = new MqttFixedHeader(MqttMessageType.DISCONNECT, false, MqttQoS.AT_MOST_ONCE, false, 0);
channel.writeAndFlush(new MqttMessage(fixedHeader2)).addListener(ChannelFutureListener.CLOSE);
//TODO: what do when the connection is closed ?
@ -99,6 +99,7 @@ final class MqttPingHandler extends ChannelInboundHandlerAdapter {
private void handlePingResp(Channel channel) {
log.trace("[{}] Handling ping response", channel.id());
if (this.pingRespTimeout != null && !this.pingRespTimeout.isCancelled() && !this.pingRespTimeout.isDone()) {
log.trace("[{}] Cancelling disconnect due to idle event because ping response was received", channel.id());
this.pingRespTimeout.cancel(true);
this.pingRespTimeout = null;
}

4
netty-mqtt/src/main/java/org/thingsboard/mqtt/PendingOperation.java

@ -17,6 +17,8 @@ package org.thingsboard.mqtt;
public interface PendingOperation {
boolean isCanceled();
boolean isCancelled();
void onMaxRetransmissionAttemptsReached();
}

101
netty-mqtt/src/main/java/org/thingsboard/mqtt/RetransmissionHandler.java

@ -18,66 +18,119 @@ package org.thingsboard.mqtt;
import io.netty.channel.EventLoop;
import io.netty.handler.codec.mqtt.MqttFixedHeader;
import io.netty.handler.codec.mqtt.MqttMessage;
import io.netty.handler.codec.mqtt.MqttMessageIdVariableHeader;
import io.netty.handler.codec.mqtt.MqttMessageType;
import io.netty.handler.codec.mqtt.MqttPublishVariableHeader;
import io.netty.handler.codec.mqtt.MqttQoS;
import io.netty.util.concurrent.ScheduledFuture;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
@Slf4j
@RequiredArgsConstructor
final class RetransmissionHandler<T extends MqttMessage> {
private volatile boolean stopped;
private final MqttClientConfig.RetransmissionConfig config;
private final PendingOperation pendingOperation;
private volatile boolean stopped;
private ScheduledFuture<?> timer;
private int timeout = 10;
private int attemptCount = 0;
@Setter
private BiConsumer<MqttFixedHeader, T> handler;
// the three fields below are used for logging only
private final String ownerId;
private String originalMessageId;
private long totalWaitingTimeMillis;
private T originalMessage;
void setOriginalMessage(T originalMessage) {
this.originalMessage = originalMessage;
var variableHeader = originalMessage.variableHeader();
if (variableHeader instanceof MqttMessageIdVariableHeader messageIdVariableHeader) {
originalMessageId = String.valueOf(messageIdVariableHeader.messageId());
} else if (variableHeader instanceof MqttPublishVariableHeader publishVariableHeader) {
originalMessageId = String.valueOf(publishVariableHeader.packetId());
} else {
originalMessageId = "N/A";
}
}
void start(EventLoop eventLoop) {
if (eventLoop == null) {
throw new NullPointerException("eventLoop");
}
if (this.handler == null) {
if (handler == null) {
throw new NullPointerException("handler");
}
this.timeout = 10;
this.startTimer(eventLoop);
log.debug("{}MessageID[{}] Starting retransmission handler", ownerId, originalMessageId);
startTimer(eventLoop);
}
private void startTimer(EventLoop eventLoop) {
if (stopped || pendingOperation.isCanceled()) {
if (stopped || pendingOperation.isCancelled()) {
return;
}
this.timer = eventLoop.schedule(() -> {
if (stopped || pendingOperation.isCanceled()) {
// Calculate the base delay using exponential backoff.
// For attemptCount == 0, delay = initial delay; for each subsequent attempt, the base delay doubles.
long baseDelay = config.initialDelayMillis() * (long) Math.pow(2, attemptCount);
// Apply jitter: random factor between (1 - jitterFactor) and (1 + jitterFactor).
double minFactor = 1.0 - config.jitterFactor();
double maxFactor = 1.0 + config.jitterFactor();
double randomFactor = config.jitterFactor() == 0 ? 1 : ThreadLocalRandom.current().nextDouble(minFactor, maxFactor);
long delayMillisWithJitter = (long) (baseDelay * randomFactor);
totalWaitingTimeMillis += delayMillisWithJitter;
timer = eventLoop.schedule(() -> {
if (stopped || pendingOperation.isCancelled()) {
return;
}
this.timeout += 5;
boolean isDup = this.originalMessage.fixedHeader().isDup();
if (this.originalMessage.fixedHeader().messageType() == MqttMessageType.PUBLISH && this.originalMessage.fixedHeader().qosLevel() != MqttQoS.AT_MOST_ONCE) {
isDup = true;
attemptCount++;
if (attemptCount > config.maxAttempts()) {
log.debug(
"{}MessageID[{}] Gave up after {} retransmission attempts; waited a total of {} ms without receiving acknowledgement",
ownerId, originalMessageId, config.maxAttempts(), totalWaitingTimeMillis
);
stop();
pendingOperation.onMaxRetransmissionAttemptsReached();
return;
}
MqttFixedHeader fixedHeader = new MqttFixedHeader(this.originalMessage.fixedHeader().messageType(), isDup, this.originalMessage.fixedHeader().qosLevel(), this.originalMessage.fixedHeader().isRetain(), this.originalMessage.fixedHeader().remainingLength());
handler.accept(fixedHeader, originalMessage);
log.debug("{}MessageID[{}] Retransmission attempt #{} out of {}", ownerId, originalMessageId, attemptCount, config.maxAttempts());
var originalFixedHeader = originalMessage.fixedHeader();
var newFixedHeader = new MqttFixedHeader(
originalFixedHeader.messageType(),
isDup(originalFixedHeader),
originalFixedHeader.qosLevel(),
originalFixedHeader.isRetain(),
originalFixedHeader.remainingLength()
);
handler.accept(newFixedHeader, originalMessage);
startTimer(eventLoop);
}, timeout, TimeUnit.SECONDS);
}, delayMillisWithJitter, TimeUnit.MILLISECONDS);
}
private static boolean isDup(MqttFixedHeader originalFixedHeader) {
return originalFixedHeader.isDup() || (originalFixedHeader.messageType() == MqttMessageType.PUBLISH && originalFixedHeader.qosLevel() != MqttQoS.AT_MOST_ONCE);
}
void stop() {
log.debug("{}MessageID[{}] Stopping retransmission handler", ownerId, originalMessageId);
stopped = true;
if (this.timer != null) {
this.timer.cancel(true);
if (timer != null) {
timer.cancel(true);
}
}
void setHandle(BiConsumer<MqttFixedHeader, T> runnable) {
this.handler = runnable;
}
void setOriginalMessage(T originalMessage) {
this.originalMessage = originalMessage;
}
}

210
netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttClientTest.java

@ -0,0 +1,210 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.mqtt;
import com.google.common.util.concurrent.Futures;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.PooledByteBufAllocator;
import io.netty.handler.codec.mqtt.MqttConnectReturnCode;
import io.netty.handler.codec.mqtt.MqttMessageType;
import io.netty.handler.codec.mqtt.MqttQoS;
import io.netty.util.ResourceLeakDetector;
import io.netty.util.concurrent.Future;
import io.netty.util.concurrent.Promise;
import lombok.extern.slf4j.Slf4j;
import org.awaitility.Awaitility;
import org.awaitility.core.ConditionTimeoutException;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.testcontainers.hivemq.HiveMQContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;
import org.thingsboard.common.util.AbstractListeningExecutor;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
@Slf4j
@Testcontainers
class MqttClientTest {
final int randomPort = 0;
@Container
HiveMQContainer broker = new HiveMQContainer(DockerImageName.parse("hivemq/hivemq-ce").withTag("2025.2"));
MqttTestProxy proxy;
MqttClient client;
AbstractListeningExecutor handlerExecutor;
@BeforeAll
static void init() {
ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID);
}
@BeforeEach
void setup() {
handlerExecutor = new AbstractListeningExecutor() {
@Override
protected int getThreadPollSize() {
return 1;
}
};
handlerExecutor.init();
}
@AfterEach
void cleanup() {
if (client != null) {
client.disconnect();
client = null;
}
if (proxy != null) {
proxy.stop();
proxy = null;
}
handlerExecutor.destroy();
handlerExecutor = null;
}
@Test
void testConnectToBroker() {
// GIVEN
var clientConfig = new MqttClientConfig();
clientConfig.setOwnerId("Test[ConnectToBroker]");
clientConfig.setClientId("connect");
client = MqttClient.create(clientConfig, null, handlerExecutor);
// WHEN
Promise<MqttConnectResult> connectFuture = client.connect(broker.getHost(), broker.getMqttPort());
// THEN
assertThat(connectFuture).isNotNull();
Awaitility.await("waiting for client to connect")
.atMost(Duration.ofSeconds(10L))
.until(connectFuture::isDone);
assertThat(connectFuture.isSuccess()).isTrue();
MqttConnectResult actualConnectResult = connectFuture.getNow();
assertThat(actualConnectResult).isNotNull();
assertThat(actualConnectResult.isSuccess()).isTrue();
assertThat(actualConnectResult.getReturnCode()).isEqualTo(MqttConnectReturnCode.CONNECTION_ACCEPTED);
assertThat(client.isConnected()).isTrue();
}
@Test
void testDisconnectDueToKeepAliveIfNoActivity() {
// GIVEN
proxy = MqttTestProxy.builder()
.localPort(randomPort)
.brokerHost(broker.getHost())
.brokerPort(broker.getMqttPort())
.brokerToClientInterceptor(msg -> msg.fixedHeader().messageType() != MqttMessageType.PINGRESP) // drop all ping responses to simulate broker down
.build();
int idleTimeoutSeconds = 2;
var clientConfig = new MqttClientConfig();
clientConfig.setOwnerId("Test[KeepAliveDisconnect]");
clientConfig.setClientId("no-activity-disconnect");
clientConfig.setTimeoutSeconds(idleTimeoutSeconds);
clientConfig.setReconnect(false); // disable auto reconnect
client = MqttClient.create(clientConfig, null, handlerExecutor);
// WHEN-THEN
connect(broker.getHost(), proxy.getPort());
// no activity...
Awaitility.await("waiting for client to disconnect")
.pollDelay(Duration.ofSeconds(idleTimeoutSeconds * 2)) // 2 seconds to wait for the first idle event and then 2 seconds for scheduled disconnect to fire
.atMost(Duration.ofSeconds(10))
.untilAsserted(() -> assertThat(client.isConnected()).isFalse());
}
@Test
void testRetransmission() {
// GIVEN
proxy = MqttTestProxy.builder()
.localPort(randomPort)
.brokerHost(broker.getHost())
.brokerPort(broker.getMqttPort())
.brokerToClientInterceptor(msg -> msg.fixedHeader().messageType() != MqttMessageType.PUBACK) // drop all pubacks to allow retransmission to happen
.build();
// create client
var clientConfig = new MqttClientConfig();
clientConfig.setOwnerId("Test[Retransmission]");
clientConfig.setClientId("retransmission");
clientConfig.setRetransmissionConfig(new MqttClientConfig.RetransmissionConfig(1, 1000L, 0d));
client = MqttClient.create(clientConfig, null, handlerExecutor);
// connect to a broker
connect(broker.getHost(), proxy.getPort());
// subscribe to a topic
String topic = "test-topic";
List<ByteBuf> receivedMessages = Collections.synchronizedList(new ArrayList<>(2));
Future<Void> subscribeFuture = client.on(topic, (__, payload) -> {
receivedMessages.add(payload);
return Futures.immediateVoidFuture();
});
Awaitility.await("waiting for client to subscribe to a topic")
.atMost(Duration.ofSeconds(10L))
.until(subscribeFuture::isDone);
// WHEN
// publish a message
ByteBuf message = PooledByteBufAllocator.DEFAULT.buffer().writeBytes("test message".getBytes(StandardCharsets.UTF_8));
client.publish(topic, message, MqttQoS.AT_LEAST_ONCE);
// THEN
// wait enough time so that retransmission happens and stops
// if retransmission works incorrectly waiting 10 seconds allows for additional retransmissions to happen
try {
Awaitility.await("wait up to 10s, stop early if too many messages")
.atMost(Duration.ofSeconds(10L))
.pollInterval(Duration.ofMillis(100))
.until(() -> receivedMessages.size() > 2);
} catch (ConditionTimeoutException __) {
// didn't exceed 2 messages
}
assertThat(receivedMessages).size().describedAs("incorrect number of messages received, expected 2 (original plus one retransmitted)").isEqualTo(2);
}
private void connect(String host, int port) {
Promise<MqttConnectResult> connectFuture = client.connect(host, port);
Awaitility.await("waiting for client to connect")
.atMost(Duration.ofSeconds(10L))
.until(connectFuture::isSuccess);
}
}

63
netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttPingHandlerTest.java

@ -1,63 +0,0 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.mqtt;
import io.netty.channel.Channel;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.DefaultEventLoop;
import io.netty.handler.timeout.IdleStateEvent;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.concurrent.TimeUnit;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.after;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class MqttPingHandlerTest {
static final int KEEP_ALIVE_SECONDS = 0;
static final int PROCESS_SEND_DISCONNECT_MSG_TIME_MS = 500;
MqttPingHandler mqttPingHandler;
@BeforeEach
void setUp() {
mqttPingHandler = new MqttPingHandler(KEEP_ALIVE_SECONDS);
}
@Test
void givenChannelReaderIdleState_whenNoPingResponse_thenDisconnectClient() throws Exception {
ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
Channel channel = mock(Channel.class);
when(ctx.channel()).thenReturn(channel);
when(channel.eventLoop()).thenReturn(new DefaultEventLoop());
ChannelFuture channelFuture = mock(ChannelFuture.class);
when(channel.writeAndFlush(any())).thenReturn(channelFuture);
mqttPingHandler.userEventTriggered(ctx, IdleStateEvent.FIRST_READER_IDLE_STATE_EVENT);
verify(
channelFuture,
after(TimeUnit.SECONDS.toMillis(KEEP_ALIVE_SECONDS) + PROCESS_SEND_DISCONNECT_MSG_TIME_MS)
).addListener(eq(ChannelFutureListener.CLOSE));
}
}

202
netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttTestProxy.java

@ -0,0 +1,202 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.mqtt;
import io.netty.bootstrap.Bootstrap;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.Channel;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.mqtt.MqttDecoder;
import io.netty.handler.codec.mqtt.MqttEncoder;
import io.netty.handler.codec.mqtt.MqttMessage;
import io.netty.util.ReferenceCountUtil;
import lombok.extern.slf4j.Slf4j;
import java.net.InetSocketAddress;
import java.util.function.Predicate;
@Slf4j
public class MqttTestProxy {
private final EventLoopGroup bossGroup;
private final EventLoopGroup workerGroup;
private Channel clientToProxyChannel;
private Channel proxyToBrokerChannel;
private final int assignedPort;
private boolean stopped;
private final Predicate<MqttMessage> brokerToClientInterceptor;
private MqttTestProxy(Builder builder) {
log.info("Starting MQTT proxy...");
brokerToClientInterceptor = builder.brokerToClientInterceptor != null ? builder.brokerToClientInterceptor : msg -> true;
bossGroup = new NioEventLoopGroup(1);
workerGroup = new NioEventLoopGroup(1);
ServerBootstrap proxyBootstrap = new ServerBootstrap();
proxyBootstrap.group(bossGroup, workerGroup)
.channel(NioServerSocketChannel.class)
.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
protected void initChannel(SocketChannel channel) {
clientToProxyChannel = channel;
clientToProxyChannel.config().setAutoRead(false); // do not accept data before we connected to a broker
connectToBroker(builder.brokerHost, builder.brokerPort).addListener(future -> {
if (future.isSuccess()) {
clientToProxyChannel.pipeline().addLast("mqttDecoder", new MqttDecoder());
clientToProxyChannel.pipeline().addLast("mqttToBroker", new MqttRelayHandler(proxyToBrokerChannel, null));
clientToProxyChannel.pipeline().addLast("mqttEncoder", MqttEncoder.INSTANCE);
clientToProxyChannel.config().setAutoRead(true); // start accepting data for a client
} else {
log.error("Failed to connect to broker", future.cause());
clientToProxyChannel.close();
}
});
}
});
try {
Channel proxyChannel = proxyBootstrap.bind(builder.localPort).sync().channel();
assignedPort = ((InetSocketAddress) proxyChannel.localAddress()).getPort();
} catch (Exception e) {
log.error("Failed to start MQTT proxy", e);
throw new RuntimeException("Failed to start MQTT proxy", e);
}
log.info("MQTT proxy started on port {}", assignedPort);
}
private ChannelFuture connectToBroker(String brokerHost, int brokerPort) {
Bootstrap proxyToBrokerBootstrap = new Bootstrap();
proxyToBrokerBootstrap.group(workerGroup)
.channel(NioSocketChannel.class)
.handler(new ChannelInitializer<SocketChannel>() {
@Override
protected void initChannel(SocketChannel channel) {
proxyToBrokerChannel = channel;
proxyToBrokerChannel.pipeline().addLast(new MqttDecoder());
proxyToBrokerChannel.pipeline().addLast("mqttToClient", new MqttRelayHandler(clientToProxyChannel, brokerToClientInterceptor));
proxyToBrokerChannel.pipeline().addLast(MqttEncoder.INSTANCE);
}
});
return proxyToBrokerBootstrap.connect(brokerHost, brokerPort);
}
private static class MqttRelayHandler extends SimpleChannelInboundHandler<MqttMessage> {
private final Channel targetChannel;
private final Predicate<MqttMessage> interceptor;
private MqttRelayHandler(Channel targetChannel, Predicate<MqttMessage> interceptor) {
this.targetChannel = targetChannel;
this.interceptor = interceptor;
}
@Override
protected void channelRead0(ChannelHandlerContext ctx, MqttMessage msg) {
log.debug("Received message: {}", msg.fixedHeader().messageType());
if (interceptor == null || interceptor.test(msg)) {
if (targetChannel.isActive()) {
targetChannel.writeAndFlush(ReferenceCountUtil.retain(msg));
}
} else {
log.info("Dropping message: {}", msg.fixedHeader().messageType());
}
}
}
public void stop() {
if (stopped) {
log.info("MQTT proxy was already stopped");
return;
}
stopped = true;
log.info("Stopping MQTT proxy...");
if (clientToProxyChannel != null) {
clientToProxyChannel.close();
}
if (proxyToBrokerChannel != null) {
proxyToBrokerChannel.close();
}
if (bossGroup != null) {
bossGroup.shutdownGracefully();
}
if (workerGroup != null) {
workerGroup.shutdownGracefully();
}
log.info("MQTT proxy stopped");
}
public int getPort() {
return assignedPort;
}
public static Builder builder() {
return new Builder();
}
public static class Builder {
private int localPort;
private String brokerHost;
private int brokerPort;
private Predicate<MqttMessage> brokerToClientInterceptor;
public Builder localPort(int localPort) {
this.localPort = localPort;
return this;
}
public Builder brokerHost(String brokerHost) {
this.brokerHost = brokerHost;
return this;
}
public Builder brokerPort(int brokerPort) {
this.brokerPort = brokerPort;
return this;
}
public Builder brokerToClientInterceptor(Predicate<MqttMessage> interceptor) {
this.brokerToClientInterceptor = interceptor;
return this;
}
public MqttTestProxy build() {
return new MqttTestProxy(this);
}
}
}

151
netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/MqttIntegrationTest.java

@ -1,151 +0,0 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.mqtt.integration;
import io.netty.buffer.Unpooled;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.handler.codec.mqtt.MqttMessageType;
import io.netty.handler.codec.mqtt.MqttQoS;
import io.netty.util.concurrent.Future;
import io.netty.util.concurrent.Promise;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.parallel.ResourceLock;
import org.thingsboard.common.util.AbstractListeningExecutor;
import org.thingsboard.mqtt.MqttClient;
import org.thingsboard.mqtt.MqttClientConfig;
import org.thingsboard.mqtt.MqttConnectResult;
import org.thingsboard.mqtt.integration.server.MqttServer;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@ResourceLock("port8885") // test MQTT server port
@Slf4j
public class MqttIntegrationTest {
static final String MQTT_HOST = "localhost";
static final int KEEPALIVE_TIMEOUT_SECONDS = 2;
static final long RECONNECT_DELAY_SECONDS = 10L;
EventLoopGroup eventLoopGroup;
MqttServer mqttServer;
MqttClient mqttClient;
AbstractListeningExecutor handlerExecutor;
@BeforeEach
public void init() throws Exception {
this.handlerExecutor = new AbstractListeningExecutor() {
@Override
protected int getThreadPollSize() {
return 4;
}
};
handlerExecutor.init();
this.eventLoopGroup = new NioEventLoopGroup();
this.mqttServer = new MqttServer();
this.mqttServer.init();
}
@AfterEach
public void destroy() throws InterruptedException {
if (this.mqttClient != null) {
this.mqttClient.disconnect();
}
if (this.mqttServer != null) {
this.mqttServer.shutdown();
}
if (this.eventLoopGroup != null) {
this.eventLoopGroup.shutdownGracefully(0, 0, TimeUnit.MILLISECONDS);
}
if (this.handlerExecutor != null) {
this.handlerExecutor.destroy();
}
}
@Test
public void givenActiveMqttClient_whenNoActivityForKeepAliveTimeout_thenDisconnectClient() throws Throwable {
//given
this.mqttClient = initClient();
log.warn("Sending publish messages...");
CountDownLatch latch = new CountDownLatch(3);
for (int i = 0; i < 3; i++) {
Thread.sleep(30);
Future<Void> pubFuture = publishMsg();
pubFuture.addListener(future -> latch.countDown());
}
log.warn("Waiting for messages acknowledgments...");
boolean awaitResult = latch.await(10, TimeUnit.SECONDS);
Assertions.assertTrue(awaitResult);
log.warn("Messages are delivered successfully...");
//when
log.warn("Starting idle period...");
Thread.sleep(5000);
//then
List<MqttMessageType> allReceivedEvents = this.mqttServer.getEventsFromClient();
long disconnectCount = allReceivedEvents.stream().filter(type -> type == MqttMessageType.DISCONNECT).count();
Assertions.assertEquals(1, disconnectCount);
}
private Future<Void> publishMsg() {
return this.mqttClient.publish(
"test/topic",
Unpooled.wrappedBuffer("payload".getBytes(StandardCharsets.UTF_8)),
MqttQoS.AT_MOST_ONCE);
}
private MqttClient initClient() throws Exception {
MqttClientConfig config = new MqttClientConfig();
config.setOwnerId("MqttIntegrationTest");
config.setTimeoutSeconds(KEEPALIVE_TIMEOUT_SECONDS);
config.setReconnectDelay(RECONNECT_DELAY_SECONDS);
MqttClient client = MqttClient.create(config, null, handlerExecutor);
client.setEventLoop(this.eventLoopGroup);
Promise<MqttConnectResult> connectFuture = client.connect(MQTT_HOST, this.mqttServer.getMqttPort());
String hostPort = MQTT_HOST + ":" + this.mqttServer.getMqttPort();
MqttConnectResult result;
try {
result = connectFuture.get(10, TimeUnit.SECONDS);
} catch (TimeoutException ex) {
connectFuture.cancel(true);
client.disconnect();
throw new RuntimeException(String.format("Failed to connect to MQTT server at %s.", hostPort));
}
if (!result.isSuccess()) {
connectFuture.cancel(true);
client.disconnect();
throw new RuntimeException(String.format("Failed to connect to MQTT server at %s. Result code is: %s", hostPort, result.getReturnCode()));
}
return client;
}
}

84
netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttServer.java

@ -1,84 +0,0 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.mqtt.integration.server;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.Channel;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.mqtt.MqttDecoder;
import io.netty.handler.codec.mqtt.MqttEncoder;
import io.netty.handler.codec.mqtt.MqttMessageType;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
@Slf4j
public class MqttServer {
@Getter
private final List<MqttMessageType> eventsFromClient = new CopyOnWriteArrayList<>();
@Getter
private final int mqttPort = 8885;
private Channel serverChannel;
private EventLoopGroup bossGroup;
private EventLoopGroup workerGroup;
public void init() throws Exception {
log.info("Starting MQTT server on port {}...", mqttPort);
bossGroup = new NioEventLoopGroup();
workerGroup = new NioEventLoopGroup();
ServerBootstrap b = new ServerBootstrap();
b.group(bossGroup, workerGroup)
.channel(NioServerSocketChannel.class)
.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
protected void initChannel(SocketChannel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
pipeline.addLast("decoder", new MqttDecoder(65536));
pipeline.addLast("encoder", MqttEncoder.INSTANCE);
MqttTransportHandler handler = new MqttTransportHandler(eventsFromClient);
pipeline.addLast(handler);
ch.closeFuture().addListener(handler);
}
})
.childOption(ChannelOption.SO_KEEPALIVE, true);
serverChannel = b.bind(mqttPort).sync().channel();
log.info("Mqtt transport started!");
}
public void shutdown() throws InterruptedException {
log.info("Stopping MQTT transport!");
try {
serverChannel.close().sync();
} finally {
workerGroup.shutdownGracefully();
bossGroup.shutdownGracefully();
}
log.info("MQTT transport stopped!");
}
}

141
netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttTransportHandler.java

@ -1,141 +0,0 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.mqtt.integration.server;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.handler.codec.mqtt.MqttConnAckMessage;
import io.netty.handler.codec.mqtt.MqttConnAckVariableHeader;
import io.netty.handler.codec.mqtt.MqttConnectMessage;
import io.netty.handler.codec.mqtt.MqttConnectReturnCode;
import io.netty.handler.codec.mqtt.MqttFixedHeader;
import io.netty.handler.codec.mqtt.MqttMessage;
import io.netty.handler.codec.mqtt.MqttMessageIdVariableHeader;
import io.netty.handler.codec.mqtt.MqttMessageType;
import io.netty.handler.codec.mqtt.MqttPubAckMessage;
import io.netty.handler.codec.mqtt.MqttPublishMessage;
import io.netty.util.ReferenceCountUtil;
import io.netty.util.concurrent.Future;
import io.netty.util.concurrent.GenericFutureListener;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
import java.util.UUID;
import static io.netty.handler.codec.mqtt.MqttMessageType.CONNACK;
import static io.netty.handler.codec.mqtt.MqttMessageType.CONNECT;
import static io.netty.handler.codec.mqtt.MqttMessageType.DISCONNECT;
import static io.netty.handler.codec.mqtt.MqttMessageType.PINGREQ;
import static io.netty.handler.codec.mqtt.MqttMessageType.PUBACK;
import static io.netty.handler.codec.mqtt.MqttMessageType.PUBLISH;
import static io.netty.handler.codec.mqtt.MqttQoS.AT_MOST_ONCE;
@Slf4j
public class MqttTransportHandler extends ChannelInboundHandlerAdapter implements GenericFutureListener<Future<? super Void>> {
private final List<MqttMessageType> eventsFromClient;
private final UUID sessionId;
MqttTransportHandler(List<MqttMessageType> eventsFromClient) {
this.sessionId = UUID.randomUUID();
this.eventsFromClient = eventsFromClient;
}
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) {
log.trace("[{}] Processing msg: {}", sessionId, msg);
try {
if (msg instanceof MqttMessage) {
MqttMessage message = (MqttMessage) msg;
if (message.decoderResult().isSuccess()) {
processMqttMsg(ctx, message);
} else {
log.error("[{}] Message decoding failed: {}", sessionId, message.decoderResult().cause().getMessage());
ctx.close();
}
} else {
log.debug("[{}] Received non mqtt message: {}", sessionId, msg.getClass().getSimpleName());
ctx.close();
}
} finally {
ReferenceCountUtil.safeRelease(msg);
}
}
void processMqttMsg(ChannelHandlerContext ctx, MqttMessage msg) {
if (msg.fixedHeader() == null) {
ctx.close();
return;
}
switch (msg.fixedHeader().messageType()) {
case CONNECT:
eventsFromClient.add(CONNECT);
processConnect(ctx, (MqttConnectMessage) msg);
break;
case DISCONNECT:
eventsFromClient.add(DISCONNECT);
ctx.close();
break;
case PUBLISH:
// QoS 0 and 1 supported only here
eventsFromClient.add(PUBLISH);
MqttPublishMessage mqttPubMsg = (MqttPublishMessage) msg;
ack(ctx, mqttPubMsg.variableHeader().packetId());
break;
case PINGREQ:
// We will not handle PINGREQ and will not send any PINGRESP to simulate the MQTT server is down
eventsFromClient.add(PINGREQ);
break;
default:
break;
}
}
void processConnect(ChannelHandlerContext ctx, MqttConnectMessage msg) {
String userName = msg.payload().userName();
String clientId = msg.payload().clientIdentifier();
log.warn("[{}][{}] Processing connect msg for client: {}!", sessionId, userName, clientId);
ctx.writeAndFlush(createMqttConnAckMsg(msg));
}
private MqttConnAckMessage createMqttConnAckMsg(MqttConnectMessage msg) {
MqttFixedHeader mqttFixedHeader =
new MqttFixedHeader(CONNACK, false, AT_MOST_ONCE, false, 0);
MqttConnAckVariableHeader mqttConnAckVariableHeader =
new MqttConnAckVariableHeader(MqttConnectReturnCode.CONNECTION_ACCEPTED, !msg.variableHeader().isCleanSession());
return new MqttConnAckMessage(mqttFixedHeader, mqttConnAckVariableHeader);
}
private void ack(ChannelHandlerContext ctx, int msgId) {
if (msgId > 0) {
ctx.writeAndFlush(createMqttPubAckMsg(msgId));
}
}
public static MqttPubAckMessage createMqttPubAckMsg(int requestId) {
MqttFixedHeader mqttFixedHeader =
new MqttFixedHeader(PUBACK, false, AT_MOST_ONCE, false, 0);
MqttMessageIdVariableHeader mqttMsgIdVariableHeader =
MqttMessageIdVariableHeader.from(requestId);
return new MqttPubAckMessage(mqttFixedHeader, mqttMsgIdVariableHeader);
}
@Override
public void operationComplete(Future<? super Void> future) {
log.trace("[{}] Channel closed!", sessionId);
}
}

3
netty-mqtt/src/test/resources/junit-platform.properties

@ -1,3 +0,0 @@
junit.jupiter.execution.parallel.enabled = true
junit.jupiter.execution.parallel.mode.default = concurrent
junit.jupiter.execution.parallel.mode.classes.default = concurrent

6
pom.xml

@ -1973,6 +1973,12 @@
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>hivemq</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-redis</artifactId>

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save