Browse Source

Merge remote-tracking branch 'upstream/master' into feature/unit-convertor

pull/13282/head
Vladyslav_Prykhodko 1 year ago
parent
commit
a925eb4380
  1. 5
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  2. 14
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
  3. 2
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java
  4. 2
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java
  5. 37
      application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientRetransmissionSettingsComponent.java
  6. 47
      application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientSettingsComponent.java
  7. 2
      application/src/main/java/org/thingsboard/server/controller/AlarmController.java
  8. 5
      application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java
  9. 80
      application/src/main/java/org/thingsboard/server/controller/TrendzController.java
  10. 3
      application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java
  11. 14
      application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java
  12. 23
      application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java
  13. 3
      application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java
  14. 2
      application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java
  15. 46
      application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java
  16. 17
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  17. 3
      application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java
  18. 8
      application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java
  19. 9
      application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java
  20. 2
      application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java
  21. 2
      application/src/main/java/org/thingsboard/server/service/ruleengine/DefaultRuleEngineCallService.java
  22. 2
      application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java
  23. 22
      application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java
  24. 9
      application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java
  25. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java
  26. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java
  27. 45
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java
  28. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java
  29. 6
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java
  30. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java
  31. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java
  32. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java
  33. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationRuleImportService.java
  34. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationTargetImportService.java
  35. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationTemplateImportService.java
  36. 20
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java
  37. 14
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java
  38. 6
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetTypeImportService.java
  39. 6
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java
  40. 2
      application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
  41. 2
      application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java
  42. 33
      application/src/main/resources/thingsboard.yml
  43. 12
      application/src/test/java/org/thingsboard/server/actors/rule/DefaultTbContextTest.java
  44. 30
      application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java
  45. 2
      application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java
  46. 4
      application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java
  47. 3
      application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java
  48. 77
      application/src/test/java/org/thingsboard/server/controller/TrendzControllerTest.java
  49. 163
      application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java
  50. 2
      application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java
  51. 2
      application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineStrategyTest.java
  52. 2
      application/src/test/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcServiceTest.java
  53. 2
      application/src/test/java/org/thingsboard/server/service/ruleengine/DefaultRuleEngineCallServiceTest.java
  54. 43
      application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java
  55. 29
      common/dao-api/src/main/java/org/thingsboard/server/dao/trendz/TrendzSettingsService.java
  56. 1
      common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java
  57. 2
      common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java
  58. 8
      common/data/src/main/java/org/thingsboard/server/common/data/TbProperty.java
  59. 1
      common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java
  60. 6
      common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java
  61. 26
      common/data/src/main/java/org/thingsboard/server/common/data/trendz/TrendzSettings.java
  62. 104
      common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java
  63. 4
      common/message/src/main/proto/tbmsg.proto
  64. 22
      common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java
  65. 8
      common/proto/src/main/proto/queue.proto
  66. 2
      common/queue/src/main/java/org/thingsboard/server/queue/common/TbRuleEngineProducerService.java
  67. 38
      common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java
  68. 17
      common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java
  69. 20
      common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java
  70. 37
      dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java
  71. 6
      dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java
  72. 69
      dao/src/main/java/org/thingsboard/server/dao/trendz/DefaultTrendzSettingsService.java
  73. 125
      dao/src/test/java/org/thingsboard/server/dao/service/RuleChainServiceTest.java
  74. 3
      dao/src/test/resources/application-test.properties
  75. 6
      edqs/src/main/resources/edqs.yml
  76. 6
      monitoring/src/main/java/org/thingsboard/monitoring/ThingsboardMonitoringApplication.java
  77. 9
      monitoring/src/main/java/org/thingsboard/monitoring/client/TbClient.java
  78. 69
      monitoring/src/main/java/org/thingsboard/monitoring/client/WsClient.java
  79. 11
      monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportInfo.java
  80. 6
      monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringTarget.java
  81. 11
      monitoring/src/main/java/org/thingsboard/monitoring/data/ServiceFailureException.java
  82. 18
      monitoring/src/main/java/org/thingsboard/monitoring/data/cmd/EntityDataUpdate.java
  83. 47
      monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java
  84. 47
      monitoring/src/main/java/org/thingsboard/monitoring/service/BaseMonitoringService.java
  85. 252
      monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringEntityService.java
  86. 16
      monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringReporter.java
  87. 118
      monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java
  88. 16
      monitoring/src/main/java/org/thingsboard/monitoring/util/ResourceUtils.java
  89. 327
      monitoring/src/main/resources/rule_chain.json
  90. 11
      monitoring/src/main/resources/tb-monitoring.yml
  91. 12
      msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java
  92. 15
      msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java
  93. 14
      msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java
  94. 5
      msa/vc-executor/src/main/resources/tb-vc-executor.yml
  95. 20
      netty-mqtt/pom.xml
  96. 24
      netty-mqtt/src/main/java/org/thingsboard/mqtt/MaxRetransmissionsReachedException.java
  97. 30
      netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttChannelHandler.java
  98. 2
      netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClient.java
  99. 20
      netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientConfig.java
  100. 82
      netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java

5
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -32,6 +32,7 @@ import org.springframework.stereotype.Component;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.rule.engine.api.DeviceStateManager;
import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.rule.engine.api.MqttClientSettings;
import org.thingsboard.rule.engine.api.NotificationCenter;
import org.thingsboard.rule.engine.api.SmsService;
import org.thingsboard.rule.engine.api.notification.SlackService;
@ -639,6 +640,10 @@ public class ActorSystemContext {
@Getter
private long cfCalculationResultTimeout;
@Autowired
@Getter
private MqttClientSettings mqttClientSettings;
@Getter
@Setter
private TbActorSystem actorSystem;

14
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java

@ -23,14 +23,15 @@ import org.bouncycastle.util.Arrays;
import org.thingsboard.common.util.DebugModeUtil;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ListeningExecutor;
import org.thingsboard.rule.engine.api.DeviceStateManager;
import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.rule.engine.api.MqttClientSettings;
import org.thingsboard.rule.engine.api.NotificationCenter;
import org.thingsboard.rule.engine.api.RuleEngineAlarmService;
import org.thingsboard.rule.engine.api.RuleEngineApiUsageStateService;
import org.thingsboard.rule.engine.api.RuleEngineAssetProfileCache;
import org.thingsboard.rule.engine.api.RuleEngineCalculatedFieldQueueService;
import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache;
import org.thingsboard.rule.engine.api.DeviceStateManager;
import org.thingsboard.rule.engine.api.RuleEngineRpcService;
import org.thingsboard.rule.engine.api.RuleEngineTelemetryService;
import org.thingsboard.rule.engine.api.ScriptEngine;
@ -234,7 +235,8 @@ public class DefaultTbContext implements TbContext {
TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(getTenantId().getId().getMostSignificantBits())
.setTenantIdLSB(getTenantId().getId().getLeastSignificantBits())
.setTbMsg(TbMsg.toByteString(tbMsg)).build();
.setTbMsgProto(TbMsg.toProto(tbMsg))
.build();
mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, callback);
}
@ -313,7 +315,7 @@ public class DefaultTbContext implements TbContext {
TransportProtos.ToRuleEngineMsg.Builder msg = TransportProtos.ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(getTenantId().getId().getMostSignificantBits())
.setTenantIdLSB(getTenantId().getId().getLeastSignificantBits())
.setTbMsg(TbMsg.toByteString(tbMsg))
.setTbMsgProto(TbMsg.toProto(tbMsg))
.addAllRelationTypes(relationTypes);
if (failureMessage != null) {
msg.setFailureMessage(failureMessage);
@ -1010,13 +1012,17 @@ public class DefaultTbContext implements TbContext {
return mainCtx.getAuditLogService();
}
@Override
public MqttClientSettings getMqttClientSettings() {
return mainCtx.getMqttClientSettings();
}
private TbMsgMetaData getActionMetaData(RuleNodeId ruleNodeId) {
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("ruleNodeId", ruleNodeId.toString());
return metaData;
}
@Override
public void schedule(Runnable runnable, long delay, TimeUnit timeUnit) {
mainCtx.getScheduler().schedule(runnable, delay, timeUnit);

2
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java

@ -381,7 +381,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
ToRuleEngineMsg toQueueMsg = ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setTbMsg(TbMsg.toByteString(newMsg))
.setTbMsgProto(TbMsg.toProto(newMsg))
.build();
clusterService.pushMsgToRuleEngine(tpi, newMsg.getId(), toQueueMsg, callbackWrapper);
}

2
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java

@ -210,7 +210,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
TransportProtos.ToRuleEngineMsg toQueueMsg = TransportProtos.ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setTbMsg(TbMsg.toByteString(tbMsg))
.setTbMsgProto(TbMsg.toProto(tbMsg))
.build();
systemContext.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), toQueueMsg, null);
defaultCtx.ack(source);

37
application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientRetransmissionSettingsComponent.java

@ -0,0 +1,37 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.config.mqtt;
import jakarta.validation.constraints.PositiveOrZero;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.validation.annotation.Validated;
@Data
@Validated
@Configuration
@ConfigurationProperties(prefix = "mqtt.client.retransmission")
public class MqttClientRetransmissionSettingsComponent {
@PositiveOrZero
private int maxAttempts;
@PositiveOrZero
private long initialDelayMillis;
@PositiveOrZero
private double jitterFactor;
}

47
application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientSettingsComponent.java

@ -0,0 +1,47 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.config.mqtt;
import lombok.EqualsAndHashCode;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
import org.springframework.context.annotation.Configuration;
import org.thingsboard.rule.engine.api.MqttClientSettings;
@ToString
@EqualsAndHashCode
@Configuration
@RequiredArgsConstructor
public class MqttClientSettingsComponent implements MqttClientSettings {
private final MqttClientRetransmissionSettingsComponent retransmissionSettingsComponent;
@Override
public int getRetransmissionMaxAttempts() {
return retransmissionSettingsComponent.getMaxAttempts();
}
@Override
public long getRetransmissionInitialDelayMillis() {
return retransmissionSettingsComponent.getInitialDelayMillis();
}
@Override
public double getRetransmissionJitterFactor() {
return retransmissionSettingsComponent.getJitterFactor();
}
}

2
application/src/main/java/org/thingsboard/server/controller/AlarmController.java

@ -157,7 +157,7 @@ public class AlarmController extends BaseController {
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
@RequestMapping(value = "/alarm/{alarmId}", method = RequestMethod.DELETE)
@ResponseBody
public Boolean deleteAlarm(@Parameter(description = ALARM_ID_PARAM_DESCRIPTION) @PathVariable(ALARM_ID) String strAlarmId) throws ThingsboardException {
public boolean deleteAlarm(@Parameter(description = ALARM_ID_PARAM_DESCRIPTION) @PathVariable(ALARM_ID) String strAlarmId) throws ThingsboardException {
checkParameter(ALARM_ID, strAlarmId);
AlarmId alarmId = new AlarmId(toUUID(strAlarmId));
Alarm alarm = checkAlarmId(alarmId, Operation.DELETE);

5
application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java

@ -42,6 +42,7 @@ import org.thingsboard.server.common.data.settings.UserSettings;
import org.thingsboard.server.common.data.settings.UserSettingsType;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.dao.mobile.QrCodeSettingService;
import org.thingsboard.server.dao.trendz.TrendzSettingsService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.model.UserPrincipal;
@ -87,6 +88,9 @@ public class SystemInfoController extends BaseController {
@Autowired
private DebugModeRateLimitsConfig debugModeRateLimitsConfig;
@Autowired
private TrendzSettingsService trendzSettingsService;
@PostConstruct
public void init() {
JsonNode info = buildInfoObject();
@ -158,6 +162,7 @@ public class SystemInfoController extends BaseController {
}
systemParams.setMaxArgumentsPerCF(tenantProfileConfiguration.getMaxArgumentsPerCF());
systemParams.setMaxDataPointsPerRollingArg(tenantProfileConfiguration.getMaxDataPointsPerRollingArg());
systemParams.setTrendzSettings(trendzSettingsService.findTrendzSettings(currentUser.getTenantId()));
}
systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID))
.map(QrCodeSettings::getQrCodeConfig).map(QRCodeConfig::isShowOnHomePage)

80
application/src/main/java/org/thingsboard/server/controller/TrendzController.java

@ -0,0 +1,80 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.controller;
import lombok.RequiredArgsConstructor;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.core.annotation.AuthenticationPrincipal;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.trendz.TrendzSettings;
import org.thingsboard.server.config.annotations.ApiOperation;
import org.thingsboard.server.dao.trendz.TrendzSettingsService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;
import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END;
import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START;
import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE;
import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH;
import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH;
@RestController
@TbCoreComponent
@RequiredArgsConstructor
@RequestMapping("/api")
public class TrendzController extends BaseController {
private final TrendzSettingsService trendzSettingsService;
@ApiOperation(value = "Save Trendz settings (saveTrendzSettings)",
notes = "Saves Trendz settings for this tenant.\n" + NEW_LINE +
"Here is an example of the Trendz settings:\n" +
MARKDOWN_CODE_BLOCK_START +
"{\n" +
" \"enabled\": true,\n" +
" \"baseUrl\": \"https://some.domain.com:18888/also_necessary_prefix\"\n" +
"}" +
MARKDOWN_CODE_BLOCK_END +
TENANT_AUTHORITY_PARAGRAPH)
@PostMapping("/trendz/settings")
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
public TrendzSettings saveTrendzSettings(@RequestBody TrendzSettings trendzSettings,
@AuthenticationPrincipal SecurityUser user) throws ThingsboardException {
accessControlService.checkPermission(user, Resource.ADMIN_SETTINGS, Operation.WRITE);
TenantId tenantId = user.getTenantId();
trendzSettingsService.saveTrendzSettings(tenantId, trendzSettings);
return trendzSettings;
}
@ApiOperation(value = "Get Trendz Settings (getTrendzSettings)",
notes = "Retrieves Trendz settings for this tenant." +
TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH)
@GetMapping("/trendz/settings")
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
public TrendzSettings getTrendzSettings(@AuthenticationPrincipal SecurityUser user) {
TenantId tenantId = user.getTenantId();
return trendzSettingsService.findTrendzSettings(tenantId);
}
}

3
application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java

@ -284,7 +284,8 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService {
protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) {
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator());
TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg))
TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder()
.setTbMsgProto(TbMsg.toProto(tbMsg))
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build();
ruleEngineMsgProducer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback);

14
application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java

@ -17,10 +17,8 @@ package org.thingsboard.server.service.entitiy;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.core.env.Environment;
import org.thingsboard.server.cluster.TbClusterService;
@ -31,16 +29,10 @@ import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.dao.alarm.AlarmService;
import org.thingsboard.server.dao.asset.AssetProfileService;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import org.thingsboard.server.service.telemetry.AlarmSubscriptionService;
@ -55,12 +47,6 @@ public abstract class AbstractTbEntityService {
@Autowired
private Environment env;
@Value("${server.log_controller_error_stack_trace}")
@Getter
private boolean logControllerErrorStackTrace;
@Autowired
protected DbCallbackExecutorService dbExecutor;
@Autowired(required = false)
protected TbLogEntityActionService logEntityActionService;
@Autowired(required = false)

23
application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java

@ -190,11 +190,24 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb
}
@Override
public Boolean delete(Alarm alarm, User user) {
TenantId tenantId = alarm.getTenantId();
logEntityActionService.logEntityAction(tenantId, alarm.getOriginator(), alarm, alarm.getCustomerId(),
ActionType.ALARM_DELETE, user, alarm.getId());
return alarmSubscriptionService.deleteAlarm(tenantId, alarm.getId());
public boolean delete(Alarm alarm, User user) {
var tenantId = alarm.getTenantId();
var alarmId = alarm.getId();
var alarmOriginator = alarm.getOriginator();
boolean deleted;
try {
deleted = alarmSubscriptionService.deleteAlarm(tenantId, alarmId);
} catch (Exception e) {
logEntityActionService.logEntityAction(tenantId, emptyId(alarmOriginator.getEntityType()), ActionType.ALARM_DELETE, user, e, alarmId);
throw e;
}
if (deleted) {
logEntityActionService.logEntityAction(tenantId, alarmOriginator, alarm, alarm.getCustomerId(), ActionType.ALARM_DELETE, user, alarmId);
}
return deleted;
}
private static long getOrDefault(long ts) {

3
application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java

@ -43,5 +43,6 @@ public interface TbAlarmService {
void unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, List<UUID> alarms, long unassignTs);
Boolean delete(Alarm alarm, User user);
boolean delete(Alarm alarm, User user);
}

2
application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java

@ -86,7 +86,7 @@ public class DefaultTbOtaPackageService extends AbstractTbEntityService implemen
otaPackage.setContentType(contentType);
otaPackage.setData(ByteBuffer.wrap(data));
otaPackage.setDataSize((long) data.length);
OtaPackageInfo savedOtaPackage = otaPackageService.saveOtaPackage(otaPackage);
OtaPackageInfo savedOtaPackage = new OtaPackageInfo(otaPackageService.saveOtaPackage(otaPackage));
logEntityActionService.logEntityAction(tenantId, savedOtaPackage.getId(), savedOtaPackage, null, actionType, user);
return savedOtaPackage;
} catch (Exception e) {

46
application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java

@ -25,21 +25,30 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.alarm.AlarmSeverity;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageDataIterable;
import org.thingsboard.server.common.data.query.DynamicValue;
import org.thingsboard.server.common.data.query.FilterPredicateValue;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.sql.JpaExecutorService;
import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.service.component.RuleNodeClassInfo;
import org.thingsboard.server.service.install.DbUpgradeExecutorService;
import org.thingsboard.server.utils.TbNodeUpgradeUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import static org.thingsboard.server.dao.rule.BaseRuleChainService.TB_RULE_CHAIN_INPUT_NODE;
@Service
@Profile("install")
@Slf4j
@ -51,19 +60,50 @@ public class DefaultDataUpdateService implements DataUpdateService {
@Autowired
private RuleChainService ruleChainService;
@Autowired
private RelationService relationService;
@Autowired
private ComponentDiscoveryService componentDiscoveryService;
@Autowired
JpaExecutorService jpaExecutorService;
private DbUpgradeExecutorService executorService;
@Override
public void updateData() throws Exception {
log.info("Updating data ...");
//TODO: should be cleaned after each release
updateInputNodes();
log.info("Data updated.");
}
private void updateInputNodes() {
log.info("Creating relations for input nodes...");
int n = 0;
var inputNodes = new PageDataIterable<>(pageLink -> ruleChainService.findAllRuleNodesByType(TB_RULE_CHAIN_INPUT_NODE, pageLink), 1024);
for (RuleNode inputNode : inputNodes) {
try {
RuleChainId targetRuleChainId = Optional.ofNullable(inputNode.getConfiguration().get("ruleChainId"))
.filter(JsonNode::isTextual).map(JsonNode::asText).map(id -> new RuleChainId(UUID.fromString(id)))
.orElse(null);
if (targetRuleChainId == null) {
continue;
}
EntityRelation relation = new EntityRelation();
relation.setFrom(inputNode.getRuleChainId());
relation.setTo(targetRuleChainId);
relation.setType(EntityRelation.USES_TYPE);
relation.setTypeGroup(RelationTypeGroup.COMMON);
relationService.saveRelation(TenantId.SYS_TENANT_ID, relation);
n++;
} catch (Exception e) {
log.error("Failed to save relation for input node: {}", inputNode, e);
}
}
log.info("Created {} relations for input nodes", n);
}
@Override
public void upgradeRuleNodes() {
int totalRuleNodesUpgraded = 0;
@ -107,7 +147,7 @@ public class DefaultDataUpdateService implements DataUpdateService {
ruleNodeId, ruleNodeType, fromVersion, toVersion);
try {
TbNodeUpgradeUtils.upgradeConfigurationAndVersion(ruleNode, ruleNodeClassInfo);
saveFutures.add(jpaExecutorService.submit(() -> {
saveFutures.add(executorService.submit(() -> {
ruleChainService.saveRuleNode(TenantId.SYS_TENANT_ID, ruleNode);
log.debug("Successfully upgrade rule node with id: {} type: {} fromVersion: {} toVersion: {}",
ruleNodeId, ruleNodeType, fromVersion, toVersion);

17
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -552,19 +552,10 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
proto.getScope(), KvProtoUtil.toAttributeKvList(proto.getDataList()), callback);
} else if (msg.hasAttrDelete()) {
TbAttributeDeleteProto proto = msg.getAttrDelete();
if (proto.hasNotifyDevice()) {
// handles old messages with deprecated 'notifyDevice'
subscriptionManagerService.onAttributesDelete(
toTenantId(proto.getTenantIdMSB(), proto.getTenantIdLSB()),
TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()),
proto.getScope(), proto.getKeysList(), proto.getNotifyDevice(), callback);
} else {
// handles new messages without 'notifyDevice'
subscriptionManagerService.onAttributesDelete(
toTenantId(proto.getTenantIdMSB(), proto.getTenantIdLSB()),
TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()),
proto.getScope(), proto.getKeysList(), callback);
}
subscriptionManagerService.onAttributesDelete(
toTenantId(proto.getTenantIdMSB(), proto.getTenantIdLSB()),
TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()),
proto.getScope(), proto.getKeysList(), callback);
} else if (msg.hasTsDelete()) {
TbTimeSeriesDeleteProto proto = msg.getTsDelete();
subscriptionManagerService.onTimeSeriesDelete(

3
application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java

@ -20,6 +20,7 @@ import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.msg.gen.MsgProtos;
import org.thingsboard.server.common.util.ProtoUtils;
import org.thingsboard.server.gen.transport.TransportProtos;
import java.util.UUID;
@ -34,7 +35,7 @@ public class SequentialByOriginatorIdTbRuleEngineSubmitStrategy extends Sequenti
@Override
protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) {
try {
MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(msg.getTbMsg());
MsgProtos.TbMsgProto proto = ProtoUtils.getTbMsgProto(msg);
return EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()));
} catch (InvalidProtocolBufferException e) {
log.warn("[{}] Failed to parse TbMsg: {}", queueName, msg);

8
application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java

@ -19,8 +19,8 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import org.thingsboard.server.common.data.queue.ProcessingStrategy;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import org.thingsboard.server.common.util.ProtoUtils;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -125,7 +125,7 @@ public class TbRuleEngineProcessingStrategyFactory {
}
log.debug("[{}] Going to reprocess {} messages", queueName, toReprocess.size());
if (log.isTraceEnabled()) {
toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(result.getQueueName(), msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, ProtoUtils.fromTbMsgProto(result.getQueueName(), msg.getValue(), TbMsgCallback.EMPTY)));
}
if (pauseBetweenRetries > 0) {
try {
@ -164,10 +164,10 @@ public class TbRuleEngineProcessingStrategyFactory {
log.debug("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size());
}
if (log.isTraceEnabled()) {
result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(result.getQueueName(), msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, ProtoUtils.fromTbMsgProto(result.getQueueName(), msg.getValue(), TbMsgCallback.EMPTY)));
}
if (log.isTraceEnabled()) {
result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, TbMsg.fromBytes(result.getQueueName(), msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, ProtoUtils.fromTbMsgProto(result.getQueueName(), msg.getValue(), TbMsgCallback.EMPTY)));
}
return new TbRuleEngineProcessingDecision(true, null);
}

9
application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java

@ -30,6 +30,7 @@ import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.common.util.ProtoUtils;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -178,7 +179,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager<T
new TbMsgPackCallback(id, tenantId, packCtx, stats.getTimer(tenantId, SUCCESSFUL_STATUS), stats.getTimer(tenantId, FAILED_STATUS)) :
new TbMsgPackCallback(id, tenantId, packCtx);
try {
if (!toRuleEngineMsg.getTbMsg().isEmpty()) {
if (!toRuleEngineMsg.getTbMsg().isEmpty() || toRuleEngineMsg.hasTbMsgProto()) {
forwardToRuleEngineActor(config.getName(), tenantId, toRuleEngineMsg, callback);
} else {
callback.onSuccess();
@ -189,7 +190,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager<T
}
private void forwardToRuleEngineActor(String queueName, TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) {
TbMsg tbMsg = TbMsg.fromBytes(queueName, toRuleEngineMsg.getTbMsg().toByteArray(), callback);
TbMsg tbMsg = ProtoUtils.fromTbMsgProto(queueName, toRuleEngineMsg, callback);
QueueToRuleEngineMsg msg;
ProtocolStringList relationTypesList = toRuleEngineMsg.getRelationTypesList();
Set<String> relationTypes;
@ -207,7 +208,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager<T
log.info("[{}] {} to process [{}] messages", queueKey, prefix, map.size());
for (Map.Entry<UUID, TbProtoQueueMsg<ToRuleEngineMsg>> pending : map.entrySet()) {
ToRuleEngineMsg tmp = pending.getValue().getValue();
TbMsg tmpMsg = TbMsg.fromBytes(config.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY);
TbMsg tmpMsg = ProtoUtils.fromTbMsgProto(config.getName(), tmp, TbMsgCallback.EMPTY);
RuleNodeInfo ruleNodeInfo = ctx.getLastVisitedRuleNode(pending.getKey());
if (printAll) {
log.trace("[{}][{}] {} to process message: {}, Last Rule Node: {}", queueKey, TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo);
@ -236,7 +237,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager<T
}
for (TbProtoQueueMsg<ToRuleEngineMsg> msg : msgs) {
try {
MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray());
MsgProtos.TbMsgProto tbMsgProto = ProtoUtils.getTbMsgProto(msg.getValue());
EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB()));
TopicPartitionInfo tpi = ctx.getPartitionService().resolve(ServiceType.TB_RULE_ENGINE, config.getName(), TenantId.SYS_TENANT_ID, originator);

2
application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java

@ -133,7 +133,7 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi
TransportProtos.RestApiCallResponseMsgProto msg = TransportProtos.RestApiCallResponseMsgProto.newBuilder()
.setRequestIdMSB(requestId.getMostSignificantBits())
.setRequestIdLSB(requestId.getLeastSignificantBits())
.setResponse(TbMsg.toByteString(tbMsg))
.setResponseProto(TbMsg.toProto(tbMsg))
.build();
clusterService.pushNotificationToCore(serviceId, msg, null);
}

2
application/src/main/java/org/thingsboard/server/service/ruleengine/DefaultRuleEngineCallService.java

@ -73,7 +73,7 @@ public class DefaultRuleEngineCallService implements RuleEngineCallService {
UUID requestId = new UUID(restApiCallResponseMsg.getRequestIdMSB(), restApiCallResponseMsg.getRequestIdLSB());
Consumer<TbMsg> consumer = requests.remove(requestId);
if (consumer != null) {
consumer.accept(TbMsg.fromBytes(null, restApiCallResponseMsg.getResponse().toByteArray(), TbMsgCallback.EMPTY));
consumer.accept(TbMsg.fromProto(null, restApiCallResponseMsg.getResponseProto(), restApiCallResponseMsg.getResponse(), TbMsgCallback.EMPTY));
} else {
log.trace("[{}] Unknown or stale rest api call response received", requestId);
}

2
application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java

@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.service.security.model.SecurityUser;
@Component(value="sysAdminPermissions")
@Component(value = "sysAdminPermissions")
public class SysAdminPermissions extends AbstractPermissions {
public SysAdminPermissions() {

22
application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java

@ -20,10 +20,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.alarm.AlarmInfo;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId;
@ -36,7 +33,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.common.msg.rule.engine.DeviceAttributesEventNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -73,7 +69,6 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene
private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueProducerProvider producerProvider;
private final TbLocalSubscriptionService localSubscriptionService;
private final TbClusterService clusterService;
private final SubscriptionSchedulerComponent scheduler;
private final Lock subsLock = new ReentrantLock();
@ -204,15 +199,14 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene
@Override
public void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List<String> keys, TbCallback callback) {
onAttributesDelete(tenantId, entityId, scope, keys, false, callback);
}
@Override
public void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List<String> keys, boolean notifyDevice, TbCallback callback) {
processAttributesUpdate(entityId, scope,
keys.stream().map(key -> new BaseAttributeKvEntry(0, new StringDataEntry(key, ""))).collect(Collectors.toList()));
if (entityId.getEntityType() == EntityType.DEVICE && TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope) && notifyDevice) {
clusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onDelete(tenantId, new DeviceId(entityId.getId()), scope, keys), null);
try {
List<AttributeKvEntry> deletedEntries = keys.stream()
.<AttributeKvEntry>map(key -> new BaseAttributeKvEntry(0L, new StringDataEntry(key, "")))
.toList();
processAttributesUpdate(entityId, scope, deletedEntries);
} catch (Exception e) {
callback.onFailure(e);
return;
}
callback.onSuccess();
}

9
application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java

@ -41,15 +41,6 @@ public interface SubscriptionManagerService extends ApplicationListener<Partitio
void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List<String> keys, TbCallback empty);
/**
* This method is retained solely for backwards compatibility, specifically to handle
* legacy proto messages that include the notifyDevice field.
*
* @deprecated as of 4.0, this method will be removed in future releases.
*/
@Deprecated(forRemoval = true, since = "4.0")
void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List<String> keys, boolean notifyDevice, TbCallback empty);
void onTimeSeriesDelete(TenantId tenantId, EntityId entityId, List<String> keys, TbCallback callback);
void onAlarmUpdate(TenantId tenantId, EntityId entityId, AlarmInfo alarm, TbCallback callback);

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java

@ -46,7 +46,7 @@ public class AssetImportService extends BaseEntityImportService<AssetId, Asset,
}
@Override
protected Asset saveOrUpdate(EntitiesImportCtx ctx, Asset asset, EntityExportData<Asset> exportData, IdProvider idProvider) {
protected Asset saveOrUpdate(EntitiesImportCtx ctx, Asset asset, EntityExportData<Asset> exportData, IdProvider idProvider, CompareResult compareResult) {
Asset savedAsset = assetService.saveAsset(asset);
if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) {
importCalculatedFields(ctx, savedAsset, exportData, idProvider);

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java

@ -49,7 +49,7 @@ public class AssetProfileImportService extends BaseEntityImportService<AssetProf
}
@Override
protected AssetProfile saveOrUpdate(EntitiesImportCtx ctx, AssetProfile assetProfile, EntityExportData<AssetProfile> exportData, IdProvider idProvider) {
protected AssetProfile saveOrUpdate(EntitiesImportCtx ctx, AssetProfile assetProfile, EntityExportData<AssetProfile> exportData, IdProvider idProvider, CompareResult compareResult) {
AssetProfile saved = assetProfileService.saveAssetProfile(assetProfile);
if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) {
importCalculatedFields(ctx, saved, exportData, idProvider);

45
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java

@ -18,6 +18,8 @@ package org.thingsboard.server.service.sync.ie.importing.impl;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.api.client.util.Objects;
import com.google.common.util.concurrent.FutureCallback;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.checkerframework.checker.nullness.qual.Nullable;
@ -117,10 +119,10 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
E prepared = prepare(ctx, entity, existingEntity, exportData, idProvider);
boolean saveOrUpdate = existingEntity == null || compare(ctx, exportData, prepared, existingEntity);
CompareResult compareResult = compare(ctx, exportData, prepared, existingEntity);
if (saveOrUpdate) {
E savedEntity = saveOrUpdate(ctx, prepared, exportData, idProvider);
if (compareResult.isUpdateNeeded()) {
E savedEntity = saveOrUpdate(ctx, prepared, exportData, idProvider, compareResult);
boolean created = existingEntity == null;
importResult.setCreated(created);
importResult.setUpdated(!created);
@ -137,6 +139,17 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
return importResult;
}
@Data
@AllArgsConstructor
static class CompareResult {
private boolean updateNeeded;
private boolean externalIdChangedOnly;
public CompareResult(boolean updateNeeded) {
this.updateNeeded = updateNeeded;
}
}
protected boolean updateRelatedEntitiesIfUnmodified(EntitiesImportCtx ctx, E prepared, D exportData, IdProvider idProvider) {
return importCalculatedFields(ctx, prepared, exportData, idProvider);
}
@ -148,18 +161,30 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
protected abstract E prepare(EntitiesImportCtx ctx, E entity, E oldEntity, D exportData, IdProvider idProvider);
protected boolean compare(EntitiesImportCtx ctx, D exportData, E prepared, E existing) {
protected CompareResult compare(EntitiesImportCtx ctx, D exportData, E prepared, E existing) {
if (existing == null) {
log.debug("[{}] Found new entity.", prepared.getId());
return new CompareResult(true);
}
var newCopy = deepCopy(prepared);
var existingCopy = deepCopy(existing);
cleanupForComparison(newCopy);
cleanupForComparison(existingCopy);
var result = !newCopy.equals(existingCopy);
if (result) {
var updateNeeded = isUpdateNeeded(ctx, exportData, newCopy, existingCopy);
boolean externalIdChangedOnly = false;
if (updateNeeded) {
log.debug("[{}] Found update.", prepared.getId());
log.debug("[{}] From: {}", prepared.getId(), newCopy);
log.debug("[{}] To: {}", prepared.getId(), existingCopy);
cleanupExternalId(newCopy);
cleanupExternalId(existingCopy);
externalIdChangedOnly = newCopy.equals(existingCopy);
}
return result;
return new CompareResult(updateNeeded, externalIdChangedOnly);
}
protected boolean isUpdateNeeded(EntitiesImportCtx ctx, D exportData, E prepared, E existing) {
return !prepared.equals(existing);
}
protected abstract E deepCopy(E e);
@ -172,7 +197,11 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
}
}
protected abstract E saveOrUpdate(EntitiesImportCtx ctx, E entity, D exportData, IdProvider idProvider);
protected void cleanupExternalId(E e) {
e.setExternalId(null);
}
protected abstract E saveOrUpdate(EntitiesImportCtx ctx, E entity, D exportData, IdProvider idProvider, CompareResult compareResult);
protected void processAfterSaved(EntitiesImportCtx ctx, EntityImportResult<E> importResult, D exportData, IdProvider idProvider) throws ThingsboardException {

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java

@ -52,7 +52,7 @@ public class CustomerImportService extends BaseEntityImportService<CustomerId, C
}
@Override
protected Customer saveOrUpdate(EntitiesImportCtx ctx, Customer customer, EntityExportData<Customer> exportData, IdProvider idProvider) {
protected Customer saveOrUpdate(EntitiesImportCtx ctx, Customer customer, EntityExportData<Customer> exportData, IdProvider idProvider, CompareResult compareResult) {
if (!customer.isPublic()) {
return customerService.saveCustomer(customer);
} else {

6
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java

@ -75,7 +75,7 @@ public class DashboardImportService extends BaseEntityImportService<DashboardId,
}
@Override
protected Dashboard saveOrUpdate(EntitiesImportCtx ctx, Dashboard dashboard, EntityExportData<Dashboard> exportData, IdProvider idProvider) {
protected Dashboard saveOrUpdate(EntitiesImportCtx ctx, Dashboard dashboard, EntityExportData<Dashboard> exportData, IdProvider idProvider, CompareResult compareResult) {
var tenantId = ctx.getTenantId();
Set<ShortCustomerInfo> assignedCustomers = Optional.ofNullable(dashboard.getAssignedCustomers()).orElse(Collections.emptySet()).stream()
@ -116,8 +116,8 @@ public class DashboardImportService extends BaseEntityImportService<DashboardId,
}
@Override
protected boolean compare(EntitiesImportCtx ctx, EntityExportData<Dashboard> exportData, Dashboard prepared, Dashboard existing) {
return super.compare(ctx, exportData, prepared, existing) || !prepared.getConfiguration().equals(existing.getConfiguration());
protected boolean isUpdateNeeded(EntitiesImportCtx ctx, EntityExportData<Dashboard> exportData, Dashboard prepared, Dashboard existing) {
return super.isUpdateNeeded(ctx, exportData, prepared, existing) || !prepared.getConfiguration().equals(existing.getConfiguration());
}
@Override

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java

@ -63,7 +63,7 @@ public class DeviceImportService extends BaseEntityImportService<DeviceId, Devic
}
@Override
protected Device saveOrUpdate(EntitiesImportCtx ctx, Device device, DeviceExportData exportData, IdProvider idProvider) {
protected Device saveOrUpdate(EntitiesImportCtx ctx, Device device, DeviceExportData exportData, IdProvider idProvider, CompareResult compareResult) {
Device savedDevice;
if (exportData.getCredentials() != null && ctx.isSaveCredentials()) {
exportData.getCredentials().setId(null);

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java

@ -51,7 +51,7 @@ public class DeviceProfileImportService extends BaseEntityImportService<DevicePr
}
@Override
protected DeviceProfile saveOrUpdate(EntitiesImportCtx ctx, DeviceProfile deviceProfile, EntityExportData<DeviceProfile> exportData, IdProvider idProvider) {
protected DeviceProfile saveOrUpdate(EntitiesImportCtx ctx, DeviceProfile deviceProfile, EntityExportData<DeviceProfile> exportData, IdProvider idProvider, CompareResult compareResult) {
DeviceProfile saved = deviceProfileService.saveDeviceProfile(deviceProfile);
if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) {
importCalculatedFields(ctx, saved, exportData, idProvider);

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java

@ -55,7 +55,7 @@ public class EntityViewImportService extends BaseEntityImportService<EntityViewI
}
@Override
protected EntityView saveOrUpdate(EntitiesImportCtx ctx, EntityView entityView, EntityExportData<EntityView> exportData, IdProvider idProvider) {
protected EntityView saveOrUpdate(EntitiesImportCtx ctx, EntityView entityView, EntityExportData<EntityView> exportData, IdProvider idProvider, CompareResult compareResult) {
return entityViewService.saveEntityView(entityView);
}

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationRuleImportService.java

@ -135,7 +135,7 @@ public class NotificationRuleImportService extends BaseEntityImportService<Notif
}
@Override
protected NotificationRule saveOrUpdate(EntitiesImportCtx ctx, NotificationRule notificationRule, EntityExportData<NotificationRule> exportData, IdProvider idProvider) {
protected NotificationRule saveOrUpdate(EntitiesImportCtx ctx, NotificationRule notificationRule, EntityExportData<NotificationRule> exportData, IdProvider idProvider, CompareResult compareResult) {
ConstraintValidator.validateFields(notificationRule);
return notificationRuleService.saveNotificationRule(ctx.getTenantId(), notificationRule);
}

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationTargetImportService.java

@ -80,7 +80,7 @@ public class NotificationTargetImportService extends BaseEntityImportService<Not
}
@Override
protected NotificationTarget saveOrUpdate(EntitiesImportCtx ctx, NotificationTarget notificationTarget, EntityExportData<NotificationTarget> exportData, IdProvider idProvider) {
protected NotificationTarget saveOrUpdate(EntitiesImportCtx ctx, NotificationTarget notificationTarget, EntityExportData<NotificationTarget> exportData, IdProvider idProvider, CompareResult compareResult) {
ConstraintValidator.validateFields(notificationTarget);
return notificationTargetService.saveNotificationTarget(ctx.getTenantId(), notificationTarget);
}

2
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationTemplateImportService.java

@ -48,7 +48,7 @@ public class NotificationTemplateImportService extends BaseEntityImportService<N
}
@Override
protected NotificationTemplate saveOrUpdate(EntitiesImportCtx ctx, NotificationTemplate notificationTemplate, EntityExportData<NotificationTemplate> exportData, IdProvider idProvider) {
protected NotificationTemplate saveOrUpdate(EntitiesImportCtx ctx, NotificationTemplate notificationTemplate, EntityExportData<NotificationTemplate> exportData, IdProvider idProvider, CompareResult compareResult) {
ConstraintValidator.validateFields(notificationTemplate);
return notificationTemplateService.saveNotificationTemplate(ctx.getTenantId(), notificationTemplate);
}

20
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java

@ -58,21 +58,29 @@ public class ResourceImportService extends BaseEntityImportService<TbResourceId,
}
@Override
protected boolean compare(EntitiesImportCtx ctx, EntityExportData<TbResource> exportData, TbResource prepared, TbResource existing) {
return true;
protected TbResource deepCopy(TbResource resource) {
return new TbResource(resource);
}
@Override
protected TbResource deepCopy(TbResource resource) {
return new TbResource(resource);
protected void cleanupForComparison(TbResource resource) {
super.cleanupForComparison(resource);
resource.setSearchText(null);
if (resource.getDescriptor().isNull()) {
resource.setDescriptor(null);
}
}
@Override
protected TbResource saveOrUpdate(EntitiesImportCtx ctx, TbResource resource, EntityExportData<TbResource> exportData, IdProvider idProvider) {
protected TbResource saveOrUpdate(EntitiesImportCtx ctx, TbResource resource, EntityExportData<TbResource> exportData, IdProvider idProvider, CompareResult compareResult) {
if (resource.getResourceType() == ResourceType.IMAGE) {
return new TbResource(imageService.saveImage(resource));
} else {
resource = resourceService.saveResource(resource);
if (compareResult.isExternalIdChangedOnly()) {
resource = resourceService.saveResource(resource, false);
} else {
resource = resourceService.saveResource(resource);
}
resource.setData(null);
resource.setPreview(null);
return resource;

14
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java

@ -18,6 +18,7 @@ package org.thingsboard.server.service.sync.ie.importing.impl;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.audit.ActionType;
@ -27,6 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.RuleChainExportData;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.rule.RuleNodeDao;
@ -103,7 +105,7 @@ public class RuleChainImportService extends BaseEntityImportService<RuleChainId,
}
@Override
protected RuleChain saveOrUpdate(EntitiesImportCtx ctx, RuleChain ruleChain, RuleChainExportData exportData, IdProvider idProvider) {
protected RuleChain saveOrUpdate(EntitiesImportCtx ctx, RuleChain ruleChain, RuleChainExportData exportData, IdProvider idProvider, CompareResult compareResult) {
ruleChain = ruleChainService.saveRuleChain(ruleChain);
if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) {
exportData.getMetaData().setRuleChainId(ruleChain.getId());
@ -115,15 +117,15 @@ public class RuleChainImportService extends BaseEntityImportService<RuleChainId,
}
@Override
protected boolean compare(EntitiesImportCtx ctx, RuleChainExportData exportData, RuleChain prepared, RuleChain existing) {
boolean different = super.compare(ctx, exportData, prepared, existing);
if (!different) {
protected boolean isUpdateNeeded(EntitiesImportCtx ctx, RuleChainExportData exportData, RuleChain prepared, RuleChain existing) {
boolean updateNeeded = super.isUpdateNeeded(ctx, exportData, prepared, existing);
if (!updateNeeded) {
RuleChainMetaData newMD = exportData.getMetaData();
RuleChainMetaData existingMD = ruleChainService.loadRuleChainMetaData(ctx.getTenantId(), prepared.getId());
existingMD.setRuleChainId(null);
different = !newMD.equals(existingMD);
updateNeeded = !newMD.equals(existingMD);
}
return different;
return updateNeeded;
}
@Override

6
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetTypeImportService.java

@ -44,13 +44,13 @@ public class WidgetTypeImportService extends BaseEntityImportService<WidgetTypeI
}
@Override
protected WidgetTypeDetails saveOrUpdate(EntitiesImportCtx ctx, WidgetTypeDetails widgetsBundle, WidgetTypeExportData exportData, IdProvider idProvider) {
protected WidgetTypeDetails saveOrUpdate(EntitiesImportCtx ctx, WidgetTypeDetails widgetsBundle, WidgetTypeExportData exportData, IdProvider idProvider, CompareResult compareResult) {
return widgetTypeService.saveWidgetType(widgetsBundle);
}
@Override
protected boolean compare(EntitiesImportCtx ctx, WidgetTypeExportData exportData, WidgetTypeDetails prepared, WidgetTypeDetails existing) {
return true;
protected CompareResult compare(EntitiesImportCtx ctx, WidgetTypeExportData exportData, WidgetTypeDetails prepared, WidgetTypeDetails existing) {
return new CompareResult(true);
}
@Override

6
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java

@ -49,7 +49,7 @@ public class WidgetsBundleImportService extends BaseEntityImportService<WidgetsB
}
@Override
protected WidgetsBundle saveOrUpdate(EntitiesImportCtx ctx, WidgetsBundle widgetsBundle, WidgetsBundleExportData exportData, IdProvider idProvider) {
protected WidgetsBundle saveOrUpdate(EntitiesImportCtx ctx, WidgetsBundle widgetsBundle, WidgetsBundleExportData exportData, IdProvider idProvider, CompareResult compareResult) {
if (CollectionsUtil.isNotEmpty(exportData.getWidgets())) {
exportData.getWidgets().forEach(widgetTypeNode -> {
String bundleAlias = widgetTypeNode.remove("bundleAlias").asText();
@ -75,8 +75,8 @@ public class WidgetsBundleImportService extends BaseEntityImportService<WidgetsB
}
@Override
protected boolean compare(EntitiesImportCtx ctx, WidgetsBundleExportData exportData, WidgetsBundle prepared, WidgetsBundle existing) {
return true;
protected CompareResult compare(EntitiesImportCtx ctx, WidgetsBundleExportData exportData, WidgetsBundle prepared, WidgetsBundle existing) {
return new CompareResult(true);
}
@Override

2
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java

@ -454,7 +454,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
}
private VersionLoadResult onError(EntityId externalId, Throwable e) {
return analyze(e, externalId).orElse(VersionLoadResult.error(EntityLoadError.runtimeError(e)));
return analyze(e, externalId).orElse(VersionLoadResult.error(EntityLoadError.runtimeError(e, externalId)));
}
private Optional<VersionLoadResult> analyze(Throwable e, EntityId externalId) {

2
application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java

@ -115,7 +115,7 @@ public class DefaultAlarmSubscriptionService extends AbstractSubscriptionService
}
@Override
public Boolean deleteAlarm(TenantId tenantId, AlarmId alarmId) {
public boolean deleteAlarm(TenantId tenantId, AlarmId alarmId) {
AlarmApiCallResult result = alarmService.delAlarm(tenantId, alarmId);
onAlarmDeleted(result);
return result.isSuccessful();

33
application/src/main/resources/thingsboard.yml

@ -644,6 +644,9 @@ cache:
mobileSecretKey:
timeToLiveInMinutes: "${CACHE_MOBILE_SECRET_KEY_TTL:2}" # QR secret key cache TTL
maxSize: "${CACHE_MOBILE_SECRET_KEY_MAX_SIZE:10000}" # 0 means the cache is disabled
trendzSettings:
timeToLiveInMinutes: "${CACHE_SPECS_TRENDZ_SETTINGS_TTL:1440}" # Trendz settings cache TTL
maxSize: "${CACHE_SPECS_TRENDZ_SETTINGS_MAX_SIZE:10000}" # 0 means the cache is disabled
# Deliberately placed outside the 'specs' group above
notificationRules:
@ -1628,6 +1631,12 @@ queue:
- key: max.poll.records
# Max poll records for edqs.state topic
value: "${TB_QUEUE_KAFKA_EDQS_STATE_MAX_POLL_RECORDS:512}"
# If you override any default Kafka topic name using environment variables, you must also specify the related consumer properties
# for the new topic in `consumer-properties-per-topic-inline`. Otherwise, the topic will not inherit its expected configuration (e.g., max.poll.records, timeouts, etc).
# Each entry sets a single property for a specific topic. To define multiple properties for a topic, repeat the topic key.
# Format: "topic1:key=value;topic1:key=value;topic2:key=value"
# Example: tb_core_updated:max.poll.records=10;tb_core_updated:bootstrap.servers=kafka1:9092,kafka2:9092;tb_edge_updated:auto.offset.reset=latest
consumer-properties-per-topic-inline: "${TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE:}"
other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000"
other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside
# - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms
@ -1952,3 +1961,27 @@ mobileApp:
googlePlayLink: "${TB_MOBILE_APP_GOOGLE_PLAY_LINK:https://play.google.com/store/apps/details?id=org.thingsboard.demo.app}"
# Link to App Store for Thingsboard Live mobile application
appStoreLink: "${TB_MOBILE_APP_APP_STORE_LINK:https://apps.apple.com/us/app/thingsboard-live/id1594355695}"
mqtt:
# MQTT client configuration parameters
client:
# Parameters that control the retransmission mechanism.
# This mechanism only applies to the handling of MQTT Publish, Subscribe, Unsubscribe and Pubrel messages.
# With the updated default settings:
# - After sending the message, wait approximately 5000 ms (± jitter) for the 1st attempt.
# - The 2nd attempt will occur after roughly 5000 * 2 = 10,000 ms (± jitter).
# - The 3rd attempt will occur after roughly 5000 * 4 = 20,000 ms (± jitter).
# - The 4th "attempt" will not actually perform a retransmission.
# Instead, the system will detect that the maximum number of attempts has been reached and drop the pending message.
retransmission:
# Maximum number of retransmission attempts allowed.
# If the attempt count exceeds this value, retransmissions will stop and the pending message will be dropped.
max_attempts: "${TB_MQTT_CLIENT_RETRANSMISSION_MAX_ATTEMPTS:3}"
# Base delay (in milliseconds) before the first retransmission attempt, measured from the moment the message is sent.
# Subsequent delays are calculated using exponential backoff.
# This base delay is also used as the reference value for applying jitter.
initial_delay_millis: "${TB_MQTT_CLIENT_RETRANSMISSION_INITIAL_DELAY_MILLIS:5000}"
# Jitter factor applied to the calculated retransmission delay.
# The actual delay is randomized within a range defined by multiplying the base delay by a factor between (1 - jitter_factor) and (1 + jitter_factor).
# For example, a jitter_factor of 0.15 means the actual delay may vary by up to ±15% of the base delay.
jitter_factor: "${TB_MQTT_CLIENT_RETRANSMISSION_JITTER_FACTOR:0.15}"

12
application/src/test/java/org/thingsboard/server/actors/rule/DefaultTbContextTest.java

@ -770,11 +770,11 @@ class DefaultTbContextTest {
ToRuleEngineMsg actualToRuleEngineMsg = toRuleEngineMsgCaptor.getValue();
assertThat(actualToRuleEngineMsg).usingRecursiveComparison()
.ignoringFields("tbMsg_")
.ignoringFields("tbMsgProto_")
.isEqualTo(ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits())
.setTenantIdLSB(TENANT_ID.getId().getLeastSignificantBits())
.setTbMsg(TbMsg.toByteString(expectedTbMsg))
.setTbMsgProto(TbMsg.toProto(expectedTbMsg))
.addAllRelationTypes(List.of(connectionType)).build());
var simpleTbQueueCallback = simpleTbQueueCallbackCaptor.getValue();
@ -827,11 +827,11 @@ class DefaultTbContextTest {
ToRuleEngineMsg actualToRuleEngineMsg = toRuleEngineMsgCaptor.getValue();
assertThat(actualToRuleEngineMsg).usingRecursiveComparison()
.ignoringFields("tbMsg_")
.ignoringFields("tbMsgProto_")
.isEqualTo(ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits())
.setTenantIdLSB(TENANT_ID.getId().getLeastSignificantBits())
.setTbMsg(TbMsg.toByteString(expectedTbMsg))
.setTbMsgProto(TbMsg.toProto(expectedTbMsg))
.build());
var simpleTbQueueCallback = simpleTbQueueCallbackCaptor.getValue();
@ -1009,11 +1009,11 @@ class DefaultTbContextTest {
ToRuleEngineMsg actualToRuleEngineMsg = toRuleEngineMsgCaptor.getValue();
assertThat(actualToRuleEngineMsg).usingRecursiveComparison()
.ignoringFields("tbMsg_")
.ignoringFields("tbMsgProto_.id_")
.isEqualTo(ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits())
.setTenantIdLSB(TENANT_ID.getId().getLeastSignificantBits())
.setTbMsg(TbMsg.toByteString(expectedTbMsg))
.setTbMsgProto(TbMsg.toProto(expectedTbMsg))
.setFailureMessage(EXCEPTION_MSG)
.addAllRelationTypes(List.of(TbNodeConnectionType.FAILURE)).build());

30
application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.controller;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.extern.slf4j.Slf4j;
@ -57,6 +58,8 @@ import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.is;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@Slf4j
@ -308,6 +311,21 @@ public class AlarmControllerTest extends AbstractControllerTest {
testNotifyEntityNever(alarm.getId(), alarm);
}
@Test
public void testDeleteNonExistentAlarm() throws Exception {
loginTenantAdmin();
var nonExistentAlarmId = Uuids.timeBased();
Mockito.reset(tbClusterService, auditLogService);
doDelete("/api/alarm/" + nonExistentAlarmId)
.andExpect(status().isNotFound())
.andExpect(statusReason(is("Alarm with id [" + nonExistentAlarmId + "] is not found")));
verifyNoInteractions(tbClusterService, auditLogService);
}
@Test
public void testClearAlarmViaCustomer() throws Exception {
loginCustomerUser();
@ -634,12 +652,12 @@ public class AlarmControllerTest extends AbstractControllerTest {
doDelete("/api/user/" + savedUser.getId().getId()).andExpect(status().isOk());
Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> {
AlarmInfo alarmInfo = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class);
Assert.assertNotNull(alarmInfo);
Assert.assertNull(alarmInfo.getAssigneeId());
Assert.assertTrue(alarmInfo.getAssignTs() >= afterAssignmentTs);
});
Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> {
AlarmInfo alarmInfo = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class);
Assert.assertNotNull(alarmInfo);
Assert.assertNull(alarmInfo.getAssigneeId());
Assert.assertTrue(alarmInfo.getAssignTs() >= afterAssignmentTs);
});
}
@Test

2
application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java

@ -216,7 +216,7 @@ public class OtaPackageControllerTest extends AbstractControllerTest {
Assert.assertEquals(CHECKSUM_ALGORITHM, savedFirmware.getChecksumAlgorithm().name());
Assert.assertEquals(CHECKSUM, savedFirmware.getChecksum());
testNotifyEntityAllOneTime(savedFirmware, savedFirmware.getId(), savedFirmware.getId(),
testNotifyEntityAllOneTime(new OtaPackageInfo(savedFirmware), savedFirmware.getId(), savedFirmware.getId(),
savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(),
ActionType.UPDATED);
}

4
application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java

File diff suppressed because one or more lines are too long

3
application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java

@ -762,7 +762,8 @@ public class TenantControllerTest extends AbstractControllerTest {
TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setTbMsg(TbMsg.toByteString(tbMsg)).build();
.setTbMsgProto(TbMsg.toProto(tbMsg))
.build();
tbClusterService.pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, null);
return tbMsg;
}

77
application/src/test/java/org/thingsboard/server/controller/TrendzControllerTest.java

@ -0,0 +1,77 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.controller;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.server.common.data.trendz.TrendzSettings;
import org.thingsboard.server.dao.service.DaoSqlTest;
import static org.assertj.core.api.Assertions.assertThat;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@DaoSqlTest
public class TrendzControllerTest extends AbstractControllerTest {
private final String trendzUrl = "https://some.domain.com:18888/also_necessary_prefix";
@Before
public void setUp() throws Exception {
loginTenantAdmin();
TrendzSettings trendzSettings = new TrendzSettings();
trendzSettings.setEnabled(true);
trendzSettings.setBaseUrl(trendzUrl);
doPost("/api/trendz/settings", trendzSettings).andExpect(status().isOk());
}
@Test
public void testTrendzSettingsWhenTenant() throws Exception {
loginTenantAdmin();
TrendzSettings trendzSettings = doGet("/api/trendz/settings", TrendzSettings.class);
assertThat(trendzSettings).isNotNull();
assertThat(trendzSettings.isEnabled()).isTrue();
assertThat(trendzSettings.getBaseUrl()).isEqualTo(trendzUrl);
String updatedUrl = "https://some.domain.com:18888/tenant_trendz";
trendzSettings.setBaseUrl(updatedUrl);
doPost("/api/trendz/settings", trendzSettings).andExpect(status().isOk());
TrendzSettings updatedTrendzSettings = doGet("/api/trendz/settings", TrendzSettings.class);
assertThat(updatedTrendzSettings).isEqualTo(trendzSettings);
}
@Test
public void testTrendzSettingsWhenCustomer() throws Exception {
loginCustomerUser();
TrendzSettings newTrendzSettings = new TrendzSettings();
newTrendzSettings.setEnabled(true);
newTrendzSettings.setBaseUrl("https://some.domain.com:18888/customer_trendz");
doPost("/api/trendz/settings", newTrendzSettings).andExpect(status().isForbidden());
TrendzSettings fetchedTrendzSettings = doGet("/api/trendz/settings", TrendzSettings.class);
assertThat(fetchedTrendzSettings).isNotNull();
assertThat(fetchedTrendzSettings.isEnabled()).isTrue();
assertThat(fetchedTrendzSettings.getBaseUrl()).isEqualTo(trendzUrl);
}
}

163
application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java

@ -15,15 +15,12 @@
*/
package org.thingsboard.server.service.entitiy.alarm;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.extern.slf4j.Slf4j;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.boot.test.mock.mockito.SpyBean;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.User;
@ -35,6 +32,9 @@ import org.thingsboard.server.common.data.alarm.AlarmInfo;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.AlarmId;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.dao.alarm.AlarmService;
@ -47,7 +47,6 @@ import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.service.entitiy.TbLogEntityActionService;
import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.security.permission.AccessControlService;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import org.thingsboard.server.service.telemetry.AlarmSubscriptionService;
@ -55,58 +54,57 @@ import org.thingsboard.server.service.telemetry.AlarmSubscriptionService;
import java.util.List;
import java.util.UUID;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
@Slf4j
@RunWith(SpringRunner.class)
@ContextConfiguration(classes = DefaultTbAlarmService.class)
@TestPropertySource(properties = {
"server.log_controller_error_stack_trace=false"
})
public class DefaultTbAlarmServiceTest {
@SpringJUnitConfig(DefaultTbAlarmService.class)
class DefaultTbAlarmServiceTest {
@MockBean
protected DbCallbackExecutorService dbExecutor;
TbLogEntityActionService logEntityActionService;
@MockBean
protected TbLogEntityActionService logEntityActionService;
EdgeService edgeService;
@MockBean
protected EdgeService edgeService;
AlarmService alarmService;
@MockBean
protected AlarmService alarmService;
TbAlarmCommentService alarmCommentService;
@MockBean
protected TbAlarmCommentService alarmCommentService;
AlarmSubscriptionService alarmSubscriptionService;
@MockBean
protected AlarmSubscriptionService alarmSubscriptionService;
CustomerService customerService;
@MockBean
protected CustomerService customerService;
TbClusterService tbClusterService;
@MockBean
protected TbClusterService tbClusterService;
EntitiesVersionControlService vcService;
@MockBean
private EntitiesVersionControlService vcService;
AccessControlService accessControlService;
@MockBean
private AccessControlService accessControlService;
TenantService tenantService;
@MockBean
private TenantService tenantService;
AssetService assetService;
@MockBean
private AssetService assetService;
DeviceService deviceService;
@MockBean
private DeviceService deviceService;
AssetProfileService assetProfileService;
@MockBean
private AssetProfileService assetProfileService;
DeviceProfileService deviceProfileService;
@MockBean
private DeviceProfileService deviceProfileService;
@MockBean
private EntityService entityService;
@SpyBean
EntityService entityService;
@Autowired
DefaultTbAlarmService service;
TenantId tenantId = TenantId.fromUUID(Uuids.timeBased());
CustomerId customerId = new CustomerId(Uuids.timeBased());
@Test
public void testSave() throws ThingsboardException {
void testSave() throws ThingsboardException {
var alarm = new AlarmInfo();
when(alarmSubscriptionService.createAlarm(any())).thenReturn(AlarmApiCallResult.builder()
.successful(true)
@ -115,45 +113,99 @@ public class DefaultTbAlarmServiceTest {
.build());
service.save(alarm, new User());
verify(logEntityActionService, times(1)).logEntityAction(any(), any(), any(), any(), eq(ActionType.ADDED), any());
verify(alarmSubscriptionService, times(1)).createAlarm(any());
verify(logEntityActionService).logEntityAction(any(), any(), any(), any(), eq(ActionType.ADDED), any());
verify(alarmSubscriptionService).createAlarm(any());
}
@Test
public void testAck() throws ThingsboardException {
void testAck() throws ThingsboardException {
var alarm = new Alarm();
when(alarmSubscriptionService.acknowledgeAlarm(any(), any(), anyLong()))
.thenReturn(AlarmApiCallResult.builder().successful(true).modified(true).alarm(new AlarmInfo()).build());
service.ack(alarm, new User(new UserId(UUID.randomUUID())));
verify(alarmCommentService, times(1)).saveAlarmComment(any(), any(), any());
verify(logEntityActionService, times(1)).logEntityAction(any(), any(), any(), any(), eq(ActionType.ALARM_ACK), any());
verify(alarmSubscriptionService, times(1)).acknowledgeAlarm(any(), any(), anyLong());
verify(alarmCommentService).saveAlarmComment(any(), any(), any());
verify(logEntityActionService).logEntityAction(any(), any(), any(), any(), eq(ActionType.ALARM_ACK), any());
verify(alarmSubscriptionService).acknowledgeAlarm(any(), any(), anyLong());
}
@Test
public void testClear() throws ThingsboardException {
void testClear() throws ThingsboardException {
var alarm = new Alarm();
alarm.setAcknowledged(true);
when(alarmSubscriptionService.clearAlarm(any(), any(), anyLong(), any()))
.thenReturn(AlarmApiCallResult.builder().successful(true).cleared(true).alarm(new AlarmInfo()).build());
service.clear(alarm, new User(new UserId(UUID.randomUUID())));
verify(alarmCommentService, times(1)).saveAlarmComment(any(), any(), any());
verify(logEntityActionService, times(1)).logEntityAction(any(), any(), any(), any(), eq(ActionType.ALARM_CLEAR), any());
verify(alarmSubscriptionService, times(1)).clearAlarm(any(), any(), anyLong(), any());
verify(alarmCommentService).saveAlarmComment(any(), any(), any());
verify(logEntityActionService).logEntityAction(any(), any(), any(), any(), eq(ActionType.ALARM_CLEAR), any());
verify(alarmSubscriptionService).clearAlarm(any(), any(), anyLong(), any());
}
@Test
public void testDelete() {
service.delete(new Alarm(), new User());
void testDelete_deleteApiReturnsTrue_shouldLogActionAndReturnTrue() {
// GIVEN
var alarmOriginator = new DeviceId(Uuids.timeBased());
var alarm = new Alarm(new AlarmId(Uuids.timeBased()));
alarm.setTenantId(tenantId);
alarm.setCustomerId(customerId);
alarm.setOriginator(alarmOriginator);
var user = new User();
when(alarmSubscriptionService.deleteAlarm(tenantId, alarm.getId())).thenReturn(true);
verify(logEntityActionService, times(1)).logEntityAction(any(), any(), any(), any(), eq(ActionType.ALARM_DELETE), any(), any());
verify(alarmSubscriptionService, times(1)).deleteAlarm(any(), any());
// WHEN
boolean actual = service.delete(alarm, user);
assertThat(actual).isTrue();
verify(logEntityActionService).logEntityAction(tenantId, alarmOriginator, alarm, alarm.getCustomerId(), ActionType.ALARM_DELETE, user, alarm.getId());
verify(alarmSubscriptionService).deleteAlarm(tenantId, alarm.getId());
}
@Test
public void testUnassignAlarm() throws ThingsboardException {
void testDelete_deleteApiReturnsFalse_shouldNotLogActionAndReturnFalse() {
// GIVEN
var alarm = new Alarm(new AlarmId(Uuids.timeBased()));
alarm.setTenantId(tenantId);
var user = new User();
// WHEN
boolean actual = service.delete(alarm, user);
assertThat(actual).isFalse();
verifyNoInteractions(logEntityActionService);
verify(alarmSubscriptionService).deleteAlarm(tenantId, alarm.getId());
}
@Test
void testDelete_deleteApiThrowsException_shouldLogFailedActionAndRethrow() {
// GIVEN
var alarmOriginator = new DeviceId(Uuids.timeBased());
var alarm = new Alarm(new AlarmId(Uuids.timeBased()));
alarm.setTenantId(tenantId);
alarm.setOriginator(alarmOriginator);
var user = new User();
var exception = new RuntimeException("failed to delete alarm");
when(alarmSubscriptionService.deleteAlarm(tenantId, alarm.getId())).thenThrow(exception);
// WHEN-THEN
assertThatThrownBy(() -> service.delete(alarm, user))
.isInstanceOf(RuntimeException.class)
.hasMessage("failed to delete alarm");
verify(logEntityActionService).logEntityAction(tenantId, new DeviceId(EntityId.NULL_UUID), ActionType.ALARM_DELETE, user, exception, alarm.getId());
verify(alarmSubscriptionService).deleteAlarm(tenantId, alarm.getId());
}
@Test
void testUnassignAlarm() throws ThingsboardException {
AlarmInfo alarm = new AlarmInfo();
alarm.setId(new AlarmId(UUID.randomUUID()));
when(alarmSubscriptionService.unassignAlarm(any(), any(), anyLong()))
@ -174,12 +226,11 @@ public class DefaultTbAlarmServiceTest {
.comment(commentNode)
.build();
verify(alarmCommentService, times(1))
.saveAlarmComment(eq(alarm), eq(expectedAlarmComment), eq(user));
verify(alarmCommentService).saveAlarmComment(eq(alarm), eq(expectedAlarmComment), eq(user));
}
@Test
public void testUnassignDeletedUserAlarms() throws ThingsboardException {
void testUnassignDeletedUserAlarms() throws ThingsboardException {
AlarmInfo alarm = new AlarmInfo();
alarm.setId(new AlarmId(UUID.randomUUID()));
@ -189,7 +240,7 @@ public class DefaultTbAlarmServiceTest {
User user = new User();
user.setEmail("testEmail@gmail.com");
user.setId(new UserId(UUID.randomUUID()));
service.unassignDeletedUserAlarms(new TenantId(UUID.randomUUID()), user.getId(), user.getTitle(), List.of(alarm.getUuidId()), System.currentTimeMillis());
service.unassignDeletedUserAlarms(tenantId, user.getId(), user.getTitle(), List.of(alarm.getUuidId()), System.currentTimeMillis());
ObjectNode commentNode = JacksonUtil.newObjectNode();
commentNode.put("subtype", "ASSIGN");
@ -200,9 +251,7 @@ public class DefaultTbAlarmServiceTest {
.comment(commentNode)
.build();
verify(alarmCommentService, times(1))
.saveAlarmComment(eq(alarm), eq(expectedAlarmComment), eq(null));
verify(alarmCommentService).saveAlarmComment(eq(alarm), eq(expectedAlarmComment), eq(null));
}
}

2
application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java

@ -737,7 +737,7 @@ public class TbRuleEngineQueueConsumerManagerTest {
.setTenantIdMSB(tenantId.getMostSignificantBits())
.setTenantIdLSB(tenantId.getLeastSignificantBits())
.addRelationTypes("Success")
.setTbMsg(TbMsg.toByteString(tbMsg))
.setTbMsgProto(TbMsg.toProto(tbMsg))
.build());
}

2
application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineStrategyTest.java

@ -263,7 +263,7 @@ public class TbRuleEngineStrategyTest {
.setTenantIdMSB(tenantId.getMostSignificantBits())
.setTenantIdLSB(tenantId.getLeastSignificantBits())
.addRelationTypes("Success")
.setTbMsg(TbMsg.toByteString(tbMsg))
.setTbMsgProto(TbMsg.toProto(tbMsg))
.build());
}

2
application/src/test/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcServiceTest.java

@ -55,7 +55,7 @@ class DefaultTbRuleEngineRpcServiceTest {
var restApiCallResponseMsgProto = TransportProtos.RestApiCallResponseMsgProto.newBuilder()
.setRequestIdMSB(requestId.getMostSignificantBits())
.setRequestIdLSB(requestId.getLeastSignificantBits())
.setResponse(TbMsg.toByteString(msg))
.setResponseProto(TbMsg.toProto(msg))
.build();
// WHEN

2
application/src/test/java/org/thingsboard/server/service/ruleengine/DefaultRuleEngineCallServiceTest.java

@ -144,7 +144,7 @@ public class DefaultRuleEngineCallServiceTest {
private TransportProtos.RestApiCallResponseMsgProto getResponse(UUID requestId, TbMsg msg) {
return TransportProtos.RestApiCallResponseMsgProto.newBuilder()
.setResponse(TbMsg.toByteString(msg))
.setResponseProto(TbMsg.toProto(msg))
.setRequestIdMSB(requestId.getMostSignificantBits())
.setRequestIdLSB(requestId.getLeastSignificantBits())
.build();

43
application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java

@ -40,6 +40,9 @@ import org.thingsboard.server.common.data.EntityView;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.HasTenantId;
import org.thingsboard.server.common.data.OtaPackage;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.TbResourceInfo;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.asset.Asset;
@ -113,6 +116,8 @@ import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import static org.thingsboard.server.controller.TbResourceControllerTest.TEST_DATA;
import static org.thingsboard.server.controller.TbResourceControllerTest.JS_TEST_FILE_NAME;
@DaoSqlTest
public class VersionControlTest extends AbstractControllerTest {
@ -648,6 +653,26 @@ public class VersionControlTest extends AbstractControllerTest {
assertThat(importedCalculatedField.getType()).isEqualTo(calculatedField.getType());
}
@Test
public void testResourceVc_sameTenant() throws Exception {
TbResourceInfo resourceInfo = createResource("Test resource");
String versionId = createVersion("resources", EntityType.TB_RESOURCE);
TbResource resource = findResource(resourceInfo.getName());
loadVersion(versionId, EntityType.TB_RESOURCE);
TbResource importedResource = findResource(resource.getName());
checkImportedEntity(tenantId1, resource, tenantId1, importedResource);
checkImportedResourceData(resource, importedResource);
}
protected void checkImportedResourceData(TbResource resource, TbResource importedResource) {
assertThat(importedResource.getName()).isEqualTo(resource.getName());
assertThat(importedResource.getData()).isEqualTo(resource.getData());
assertThat(importedResource.getResourceKey()).isEqualTo(resource.getResourceKey());
assertThat(importedResource.getResourceType()).isEqualTo(resource.getResourceType());
}
private <E extends ExportableEntity<?> & HasTenantId> void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, E importedEntity) {
assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1);
assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2);
@ -1128,4 +1153,22 @@ public class VersionControlTest extends AbstractControllerTest {
return doGetTypedWithPageLink("/api/" + entityId.getEntityType() + "/" + entityId.getId() + "/calculatedFields?", new TypeReference<PageData<CalculatedField>>() {}, new PageLink(100, 0)).getData();
}
private TbResourceInfo createResource(String name) {
TbResource resource = new TbResource();
resource.setResourceType(ResourceType.JKS);
resource.setTitle(name);
resource.setFileName(JS_TEST_FILE_NAME);
resource.setEncodedData(TEST_DATA);
return saveTbResource(resource);
}
private TbResourceInfo saveTbResource(TbResource tbResource) {
return doPost("/api/resource", tbResource, TbResourceInfo.class);
}
private TbResource findResource(String name) throws Exception {
return doGetTypedWithPageLink("/api/resource?", new TypeReference<PageData<TbResource>>() {}, new PageLink(100, 0, name)).getData().get(0);
}
}

29
common/dao-api/src/main/java/org/thingsboard/server/dao/trendz/TrendzSettingsService.java

@ -0,0 +1,29 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.trendz;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.trendz.TrendzSettings;
public interface TrendzSettingsService {
void saveTrendzSettings(TenantId tenantId, TrendzSettings settings);
TrendzSettings findTrendzSettings(TenantId tenantId);
void deleteTrendzSettings(TenantId tenantId);
}

1
common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java

@ -35,6 +35,7 @@ public class CacheConstants {
public static final String DEVICE_PROFILE_CACHE = "deviceProfiles";
public static final String NOTIFICATION_SETTINGS_CACHE = "notificationSettings";
public static final String SENT_NOTIFICATIONS_CACHE = "sentNotifications";
public static final String TRENDZ_SETTINGS_CACHE = "trendzSettings";
public static final String ASSET_PROFILE_CACHE = "assetProfiles";
public static final String ATTRIBUTES_CACHE = "attributes";

2
common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java

@ -17,6 +17,7 @@ package org.thingsboard.server.common.data;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.Data;
import org.thingsboard.server.common.data.trendz.TrendzSettings;
import java.util.List;
@ -37,4 +38,5 @@ public class SystemParams {
String calculatedFieldDebugPerTenantLimitsConfiguration;
long maxArgumentsPerCF;
long maxDataPointsPerRollingArg;
TrendzSettings trendzSettings;
}

8
common/data/src/main/java/org/thingsboard/server/common/data/TbProperty.java

@ -15,14 +15,16 @@
*/
package org.thingsboard.server.common.data;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* Created by ashvayka on 25.09.18.
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class TbProperty {
private String key;
private String value;
}

1
common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java

@ -43,6 +43,7 @@ public class EntityRelation implements HasVersion, Serializable, EdqsObject {
public static final String EDGE_TYPE = "ManagedByEdge";
public static final String CONTAINS_TYPE = "Contains";
public static final String MANAGES_TYPE = "Manages";
public static final String USES_TYPE = "Uses";
@Setter
private EntityId from;

6
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java

@ -45,11 +45,15 @@ public class EntityLoadError implements Serializable {
}
public static EntityLoadError runtimeError(Throwable e) {
return runtimeError(e, null);
}
public static EntityLoadError runtimeError(Throwable e, EntityId externalId) {
String message = e.getMessage();
if (StringUtils.isEmpty(message)) {
message = "unexpected error (" + ClassUtils.getShortClassName(e.getClass()) + ")";
}
return EntityLoadError.builder().type("RUNTIME").message(message).build();
return EntityLoadError.builder().type("RUNTIME").message(message).source(externalId).build();
}
}

26
common/data/src/main/java/org/thingsboard/server/common/data/trendz/TrendzSettings.java

@ -0,0 +1,26 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.trendz;
import lombok.Data;
@Data
public class TrendzSettings {
private boolean enabled;
private String baseUrl;
}

104
common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java

@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.msg.gen.MsgProtos;
import org.thingsboard.server.common.msg.gen.MsgProtos.TbMsgProto;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import java.io.Serializable;
@ -151,12 +152,8 @@ public final class TbMsg implements Serializable {
this.callback = Objects.requireNonNullElse(callback, TbMsgCallback.EMPTY);
}
public static ByteString toByteString(TbMsg msg) {
return ByteString.copyFrom(toByteArray(msg));
}
public static byte[] toByteArray(TbMsg msg) {
MsgProtos.TbMsgProto.Builder builder = MsgProtos.TbMsgProto.newBuilder();
public static TbMsgProto toProto(TbMsg msg) {
TbMsgProto.Builder builder = TbMsgProto.newBuilder();
builder.setId(msg.getId().toString());
builder.setTs(msg.getTs());
builder.setType(msg.getType());
@ -205,56 +202,55 @@ public final class TbMsg implements Serializable {
}
builder.setCtx(msg.ctx.toProto());
return builder.build().toByteArray();
return builder.build();
}
public static TbMsg fromBytes(String queueName, byte[] data, TbMsgCallback callback) {
@Deprecated(forRemoval = true, since = "4.1") // to be removed in 4.2
public static TbMsg fromProto(String queueName, TbMsgProto proto, ByteString data, TbMsgCallback callback) {
try {
MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(data);
TbMsgMetaData metaData = new TbMsgMetaData(proto.getMetaData().getDataMap());
EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()));
CustomerId customerId = null;
RuleChainId ruleChainId = null;
RuleNodeId ruleNodeId = null;
UUID correlationId = null;
Integer partition = null;
List<CalculatedFieldId> calculatedFieldIds = new CopyOnWriteArrayList<>();
if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) {
customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB()));
}
if (proto.getRuleChainIdMSB() != 0L && proto.getRuleChainIdLSB() != 0L) {
ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB()));
}
if (proto.getRuleNodeIdMSB() != 0L && proto.getRuleNodeIdLSB() != 0L) {
ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB()));
}
if (proto.getCorrelationIdMSB() != 0L && proto.getCorrelationIdLSB() != 0L) {
correlationId = new UUID(proto.getCorrelationIdMSB(), proto.getCorrelationIdLSB());
partition = proto.getPartition();
}
for (MsgProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) {
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(
cfIdProto.getCalculatedFieldIdMSB(),
cfIdProto.getCalculatedFieldIdLSB()
));
calculatedFieldIds.add(calculatedFieldId);
if (!data.isEmpty()) {
proto = TbMsgProto.parseFrom(data);
}
TbMsgProcessingCtx ctx;
if (proto.hasCtx()) {
ctx = TbMsgProcessingCtx.fromProto(proto.getCtx());
} else {
// Backward compatibility with unprocessed messages fetched from queue after update.
ctx = new TbMsgProcessingCtx(proto.getRuleNodeExecCounter());
}
TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()];
return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), null, proto.getType(), entityId, customerId,
metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback);
} catch (InvalidProtocolBufferException e) {
throw new IllegalStateException("Could not parse protobuf for TbMsg", e);
}
return fromProto(queueName, proto, callback);
}
public static TbMsg fromProto(String queueName, TbMsgProto proto, TbMsgCallback callback) {
TbMsgMetaData metaData = new TbMsgMetaData(proto.getMetaData().getDataMap());
EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()));
CustomerId customerId = null;
RuleChainId ruleChainId = null;
RuleNodeId ruleNodeId = null;
UUID correlationId = null;
Integer partition = null;
List<CalculatedFieldId> calculatedFieldIds = new CopyOnWriteArrayList<>();
if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) {
customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB()));
}
if (proto.getRuleChainIdMSB() != 0L && proto.getRuleChainIdLSB() != 0L) {
ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB()));
}
if (proto.getRuleNodeIdMSB() != 0L && proto.getRuleNodeIdLSB() != 0L) {
ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB()));
}
if (proto.getCorrelationIdMSB() != 0L && proto.getCorrelationIdLSB() != 0L) {
correlationId = new UUID(proto.getCorrelationIdMSB(), proto.getCorrelationIdLSB());
partition = proto.getPartition();
}
for (MsgProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) {
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(
cfIdProto.getCalculatedFieldIdMSB(),
cfIdProto.getCalculatedFieldIdLSB()
));
calculatedFieldIds.add(calculatedFieldId);
}
TbMsgProcessingCtx ctx = TbMsgProcessingCtx.fromProto(proto.getCtx());
TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()];
return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), null, proto.getType(), entityId, customerId,
metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback);
}
public int getAndIncrementRuleNodeCounter() {
@ -504,11 +500,11 @@ public final class TbMsg implements Serializable {
public String toString() {
return "TbMsg.TbMsgBuilder(queueName=" + this.queueName + ", id=" + this.id + ", ts=" + this.ts +
", type=" + this.type + ", internalType=" + this.internalType + ", originator=" + this.originator +
", customerId=" + this.customerId + ", metaData=" + this.metaData + ", dataType=" + this.dataType +
", data=" + this.data + ", ruleChainId=" + this.ruleChainId + ", ruleNodeId=" + this.ruleNodeId +
", correlationId=" + this.correlationId + ", partition=" + this.partition + ", previousCalculatedFields=" + this.previousCalculatedFieldIds +
", ctx=" + this.ctx + ", callback=" + this.callback + ")";
", type=" + this.type + ", internalType=" + this.internalType + ", originator=" + this.originator +
", customerId=" + this.customerId + ", metaData=" + this.metaData + ", dataType=" + this.dataType +
", data=" + this.data + ", ruleChainId=" + this.ruleChainId + ", ruleNodeId=" + this.ruleNodeId +
", correlationId=" + this.correlationId + ", partition=" + this.partition + ", previousCalculatedFields=" + this.previousCalculatedFieldIds +
", ctx=" + this.ctx + ", callback=" + this.callback + ")";
}
}

4
common/message/src/main/proto/tbmsg.proto

@ -59,8 +59,8 @@ message TbMsgProto {
string data = 14;
int64 ts = 15;
// Will be removed in 3.4. Moved to processing context
int32 ruleNodeExecCounter = 16;
// ruleNodeExecCounter (16) was removed in 4.1
int64 customerIdMSB = 17;
int64 customerIdLSB = 18;

22
common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java

@ -18,6 +18,8 @@ package org.thingsboard.server.common.util;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
@ -76,12 +78,15 @@ import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.DeviceCredentialsType;
import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg;
import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg;
import org.thingsboard.server.common.msg.edge.EdgeHighPriorityMsg;
import org.thingsboard.server.common.msg.edge.FromEdgeSyncResponse;
import org.thingsboard.server.common.msg.edge.ToEdgeSyncRequest;
import org.thingsboard.server.common.msg.gen.MsgProtos;
import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponseActorMsg;
import org.thingsboard.server.common.msg.rpc.RemoveRpcActorMsg;
@ -93,8 +98,8 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceDeleteMsg;
import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg;
import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto;
import org.thingsboard.server.gen.transport.TransportProtos.ApiUsageRecordKeyProto;
import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto;
import java.util.ArrayList;
import java.util.Arrays;
@ -1344,6 +1349,21 @@ public class ProtoUtils {
return builder.build();
}
@Deprecated(forRemoval = true, since = "4.1")
public static MsgProtos.TbMsgProto getTbMsgProto(TransportProtos.ToRuleEngineMsg ruleEngineMsg) throws InvalidProtocolBufferException {
if (ruleEngineMsg.getTbMsg().isEmpty()) {
return ruleEngineMsg.getTbMsgProto();
} else {
return MsgProtos.TbMsgProto.parseFrom(ruleEngineMsg.getTbMsg());
}
}
@SneakyThrows
@Deprecated(forRemoval = true, since = "4.1") // inline to TbMsg.fromProto(queueName, ruleEngineMsg.getTbMsgProto(), callback)
public static TbMsg fromTbMsgProto(String queueName, TransportProtos.ToRuleEngineMsg ruleEngineMsg, TbMsgCallback callback) {
return TbMsg.fromProto(queueName, getTbMsgProto(ruleEngineMsg), callback);
}
private static boolean isNotNull(Object obj) {
return obj != null;
}

8
common/proto/src/main/proto/queue.proto

@ -20,6 +20,8 @@ package transport;
option java_package = "org.thingsboard.server.gen.transport";
option java_outer_classname = "TransportProtos";
import "tbmsg.proto";
/**
* Common data structures
*/
@ -125,7 +127,8 @@ message SessionInfoProto {
message RestApiCallResponseMsgProto {
int64 requestIdMSB = 1;
int64 requestIdLSB = 2;
bytes response = 5;
bytes response = 5 [deprecated = true];
msgqueue.TbMsgProto responseProto = 6;
}
enum SessionEvent {
@ -1700,9 +1703,10 @@ message ToCalculatedFieldNotificationMsg {
message ToRuleEngineMsg {
int64 tenantIdMSB = 1;
int64 tenantIdLSB = 2;
bytes tbMsg = 3;
bytes tbMsg = 3 [deprecated = true]; // for removal in 4.2
repeated string relationTypes = 4;
string failureMessage = 5;
msgqueue.TbMsgProto tbMsgProto = 6;
}
message ToRuleEngineNotificationMsg {

2
common/queue/src/main/java/org/thingsboard/server/queue/common/TbRuleEngineProducerService.java

@ -65,7 +65,7 @@ public class TbRuleEngineProducerService {
log.trace("[{}][{}] Pushing to topic {} message {}", tenantId, tbMsg.getOriginator(), tpi.getFullTopicName(), tbMsg);
}
ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder()
.setTbMsg(TbMsg.toByteString(tbMsg))
.setTbMsgProto(TbMsg.toProto(tbMsg))
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build();
producer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback);

38
common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.queue.kafka;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.Getter;
import lombok.Setter;
@ -36,7 +37,8 @@ import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.TbProperty;
import org.thingsboard.server.queue.util.PropertyUtils;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@ -138,15 +140,26 @@ public class TbKafkaSettings {
@Value("${queue.kafka.other-inline:}")
private String otherInline;
@Value("${queue.kafka.consumer-properties-per-topic-inline:}")
private String consumerPropertiesPerTopicInline;
@Deprecated
@Setter
private List<TbProperty> other;
@Setter
private Map<String, List<TbProperty>> consumerPropertiesPerTopic = Collections.emptyMap();
private Map<String, List<TbProperty>> consumerPropertiesPerTopic = new HashMap<>();
private volatile AdminClient adminClient;
@PostConstruct
public void initInlineTopicProperties() {
Map<String, List<TbProperty>> inlineProps = parseTopicPropertyList(consumerPropertiesPerTopicInline);
if (!inlineProps.isEmpty()) {
consumerPropertiesPerTopic.putAll(inlineProps);
}
}
public Properties toConsumerProps(String topic) {
Properties props = toProps();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
@ -245,6 +258,27 @@ public class TbKafkaSettings {
return props;
}
private Map<String, List<TbProperty>> parseTopicPropertyList(String inlineProperties) {
Map<String, List<String>> grouped = PropertyUtils.getGroupedProps(inlineProperties);
Map<String, List<TbProperty>> result = new HashMap<>();
grouped.forEach((topic, entries) -> {
Map<String, String> merged = new LinkedHashMap<>();
for (String entry : entries) {
String[] kv = entry.split("=", 2);
if (kv.length == 2) {
merged.put(kv[0].trim(), kv[1].trim());
}
}
List<TbProperty> props = merged.entrySet().stream()
.map(e -> new TbProperty(e.getKey(), e.getValue()))
.toList();
result.put(topic, props);
});
return result;
}
@PreDestroy
private void destroy() {
if (adminClient != null) {

17
common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java

@ -17,7 +17,9 @@ package org.thingsboard.server.queue.util;
import org.thingsboard.server.common.data.StringUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
@ -38,6 +40,21 @@ public class PropertyUtils {
return configs;
}
public static Map<String, List<String>> getGroupedProps(String properties) {
Map<String, List<String>> configs = new HashMap<>();
if (StringUtils.isNotEmpty(properties)) {
for (String property : properties.split(";")) {
if (StringUtils.isNotEmpty(property)) {
int delimiterPosition = property.indexOf(":");
String topic = property.substring(0, delimiterPosition).trim();
String value = property.substring(delimiterPosition + 1).trim();
configs.computeIfAbsent(topic, k -> new ArrayList<>()).add(value);
}
}
}
return configs;
}
public static Map<String, String> getProps(Map<String, String> defaultProperties, String propertiesStr) {
return getProps(defaultProperties, propertiesStr, PropertyUtils::getProps);
}

20
common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java

@ -33,6 +33,12 @@ import static org.mockito.Mockito.spy;
"queue.type=kafka",
"queue.kafka.bootstrap.servers=localhost:9092",
"queue.kafka.other-inline=metrics.recording.level:INFO;metrics.sample.window.ms:30000",
"queue.kafka.consumer-properties-per-topic-inline=" +
"tb_core_updated:max.poll.records=10;" +
"tb_core_updated:enable.auto.commit=true;" +
"tb_core_updated:bootstrap.servers=kafka1:9092,kafka2:9092;" +
"tb_edge_updated:max.poll.records=5;" +
"tb_edge_updated:auto.offset.reset=latest"
})
class TbKafkaSettingsTest {
@ -79,4 +85,16 @@ class TbKafkaSettingsTest {
Mockito.verify(settings).configureSSL(any());
}
}
@Test
void givenMultipleTopicsInInlineConfig_whenParsed_thenEachTopicGetsExpectedProperties() {
Properties coreProps = settings.toConsumerProps("tb_core_updated");
assertThat(coreProps.getProperty("max.poll.records")).isEqualTo("10");
assertThat(coreProps.getProperty("enable.auto.commit")).isEqualTo("true");
assertThat(coreProps.getProperty("bootstrap.servers")).isEqualTo("kafka1:9092,kafka2:9092");
Properties edgeProps = settings.toConsumerProps("tb_edge_updated");
assertThat(edgeProps.getProperty("max.poll.records")).isEqualTo("5");
assertThat(edgeProps.getProperty("auto.offset.reset")).isEqualTo("latest");
}
}

37
dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java

@ -76,6 +76,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -204,11 +205,17 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC
}
}
}
RuleChainId ruleChainId = ruleChain.getId();
List<RuleNodeUpdateResult> updatedRuleNodes = new ArrayList<>();
List<RuleNode> existingRuleNodes = getRuleChainNodes(tenantId, ruleChainMetaData.getRuleChainId());
for (RuleNode existingNode : existingRuleNodes) {
relationService.deleteEntityRelations(tenantId, existingNode.getId());
if (existingNode.getType().equals(TB_RULE_CHAIN_INPUT_NODE)) {
EntityRelation relation = getRuleChainInputRelation(ruleChainId, existingNode);
if (relation != null) {
relationService.deleteRelation(tenantId, relation);
}
}
Integer index = ruleNodeIndexMap.get(existingNode.getId());
RuleNode newRuleNode = null;
if (index != null) {
@ -220,7 +227,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC
}
updatedRuleNodes.add(new RuleNodeUpdateResult(existingNode, newRuleNode));
}
RuleChainId ruleChainId = ruleChain.getId();
if (nodes != null) {
long now = System.currentTimeMillis();
for (RuleNode node : toAddOrUpdate) {
@ -233,6 +240,12 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC
RuleNode savedNode = ruleNodeDao.save(tenantId, node);
relations.add(new EntityRelation(ruleChainMetaData.getRuleChainId(), savedNode.getId(),
EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN));
if (node.getType().equals(TB_RULE_CHAIN_INPUT_NODE)) {
EntityRelation relation = getRuleChainInputRelation(ruleChainId, node);
if (relation != null) {
relations.add(relation);
}
}
int index = nodes.indexOf(node);
nodes.set(index, savedNode);
ruleNodeIndexMap.put(savedNode.getId(), index);
@ -247,7 +260,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC
firstRuleNodeId = nodes.get(ruleChainMetaData.getFirstNodeIndex()).getId();
}
if ((ruleChain.getFirstRuleNodeId() != null && !ruleChain.getFirstRuleNodeId().equals(firstRuleNodeId))
|| (ruleChain.getFirstRuleNodeId() == null && firstRuleNodeId != null)) {
|| (ruleChain.getFirstRuleNodeId() == null && firstRuleNodeId != null)) {
ruleChain.setFirstRuleNodeId(firstRuleNodeId);
}
if (ruleChainMetaData.getConnections() != null) {
@ -265,7 +278,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC
RuleNode targetNode = new RuleNode();
targetNode.setName(targetRuleChain != null ? targetRuleChain.getName() : "Rule Chain Input");
targetNode.setRuleChainId(ruleChainId);
targetNode.setType("org.thingsboard.rule.engine.flow.TbRuleChainInputNode");
targetNode.setType(TB_RULE_CHAIN_INPUT_NODE);
var configuration = JacksonUtil.newObjectNode();
configuration.put("ruleChainId", targetRuleChainId.getId().toString());
targetNode.setConfiguration(configuration);
@ -302,6 +315,22 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC
return RuleChainUpdateResult.successful(updatedRuleNodes);
}
private EntityRelation getRuleChainInputRelation(RuleChainId ruleChainId, RuleNode inputNode) {
RuleChainId targetRuleChainId = Optional.ofNullable(inputNode.getConfiguration().get("ruleChainId"))
.filter(JsonNode::isTextual).map(JsonNode::asText).map(id -> new RuleChainId(UUID.fromString(id)))
.orElse(null);
if (targetRuleChainId != null) {
EntityRelation relation = new EntityRelation();
relation.setFrom(ruleChainId);
relation.setTo(targetRuleChainId);
relation.setType(EntityRelation.USES_TYPE);
relation.setTypeGroup(RelationTypeGroup.COMMON);
return relation;
} else {
return null;
}
}
@Override
public RuleChainMetaData loadRuleChainMetaData(TenantId tenantId, RuleChainId ruleChainId) {
Validator.validateId(ruleChainId, "Incorrect rule chain id.");

6
dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java

@ -44,6 +44,7 @@ import org.thingsboard.server.dao.service.PaginatedRemover;
import org.thingsboard.server.dao.service.Validator;
import org.thingsboard.server.dao.service.validator.TenantDataValidator;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.dao.trendz.TrendzSettingsService;
import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
import org.thingsboard.server.dao.user.UserService;
@ -81,6 +82,8 @@ public class TenantServiceImpl extends AbstractCachedEntityService<TenantId, Ten
@Autowired
private QrCodeSettingService qrCodeSettingService;
@Autowired
private TrendzSettingsService trendzSettingsService;
@Autowired
private TenantDataValidator tenantValidator;
@Autowired
protected TbTransactionalCache<TenantId, Boolean> existsTenantCache;
@ -163,9 +166,10 @@ public class TenantServiceImpl extends AbstractCachedEntityService<TenantId, Ten
Validator.validateId(tenantId, id -> INCORRECT_TENANT_ID + id);
userService.deleteAllByTenantId(tenantId);
notificationSettingsService.deleteNotificationSettings(tenantId);
trendzSettingsService.deleteTrendzSettings(tenantId);
adminSettingsService.deleteAdminSettingsByTenantId(tenantId);
qrCodeSettingService.deleteByTenantId(tenantId);
notificationSettingsService.deleteNotificationSettings(tenantId);
tenantDao.removeById(tenantId, tenantId.getId());
publishEvictEvent(new TenantEvictEvent(tenantId, true));

69
dao/src/main/java/org/thingsboard/server/dao/trendz/DefaultTrendzSettingsService.java

@ -0,0 +1,69 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.trendz;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.CacheConstants;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.trendz.TrendzSettings;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import java.util.Optional;
@Service
@RequiredArgsConstructor
@Slf4j
public class DefaultTrendzSettingsService implements TrendzSettingsService {
private final AdminSettingsService adminSettingsService;
private static final String SETTINGS_KEY = "trendz";
@CacheEvict(cacheNames = CacheConstants.TRENDZ_SETTINGS_CACHE, key = "#tenantId")
@Override
public void saveTrendzSettings(TenantId tenantId, TrendzSettings settings) {
AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByTenantIdAndKey(tenantId, SETTINGS_KEY))
.orElseGet(() -> {
AdminSettings newAdminSettings = new AdminSettings();
newAdminSettings.setTenantId(tenantId);
newAdminSettings.setKey(SETTINGS_KEY);
return newAdminSettings;
});
adminSettings.setJsonValue(JacksonUtil.valueToTree(settings));
adminSettingsService.saveAdminSettings(tenantId, adminSettings);
}
@Cacheable(cacheNames = CacheConstants.TRENDZ_SETTINGS_CACHE, key = "#tenantId")
@Override
public TrendzSettings findTrendzSettings(TenantId tenantId) {
return Optional.ofNullable(adminSettingsService.findAdminSettingsByTenantIdAndKey(tenantId, SETTINGS_KEY))
.map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), TrendzSettings.class))
.orElseGet(TrendzSettings::new);
}
@CacheEvict(cacheNames = CacheConstants.TRENDZ_SETTINGS_CACHE, key = "#tenantId")
@Override
public void deleteTrendzSettings(TenantId tenantId) {
adminSettingsService.deleteAdminSettingsByTenantIdAndKey(tenantId, SETTINGS_KEY);
}
}

125
dao/src/test/java/org/thingsboard/server/dao/service/RuleChainServiceTest.java

@ -28,12 +28,14 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.rule.RuleChainService;
import java.io.IOException;
@ -43,7 +45,10 @@ import java.util.List;
import java.util.UUID;
import java.util.function.Function;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.thingsboard.server.common.data.relation.EntityRelation.USES_TYPE;
import static org.thingsboard.server.dao.rule.BaseRuleChainService.TB_RULE_CHAIN_INPUT_NODE;
/**
* Created by igor on 3/13/18.
@ -55,6 +60,8 @@ public class RuleChainServiceTest extends AbstractServiceTest {
EdgeService edgeService;
@Autowired
RuleChainService ruleChainService;
@Autowired
RelationService relationService;
private IdComparator<RuleChain> idComparator = new IdComparator<>();
private IdComparator<RuleNode> ruleNodeIdComparator = new IdComparator<>();
@ -270,7 +277,7 @@ public class RuleChainServiceTest extends AbstractServiceTest {
List<RuleNode> ruleNodes = savedRuleChainMetaData.getNodes();
int name3Index = -1;
for (int i=0;i<ruleNodes.size();i++) {
for (int i = 0; i < ruleNodes.size(); i++) {
if ("name3".equals(ruleNodes.get(i).getName())) {
name3Index = i;
break;
@ -354,6 +361,80 @@ public class RuleChainServiceTest extends AbstractServiceTest {
Assert.assertTrue(ruleChainById.isRoot());
}
@Test
public void testSaveRuleChainWithInputNode() {
RuleChain fromRuleChain = new RuleChain();
fromRuleChain.setName("From RuleChain");
fromRuleChain.setTenantId(tenantId);
fromRuleChain = ruleChainService.saveRuleChain(fromRuleChain);
RuleChainId fromRuleChainId = fromRuleChain.getId();
RuleChain toRuleChain1 = new RuleChain();
toRuleChain1.setName("To Rule Chain 1");
toRuleChain1.setTenantId(tenantId);
toRuleChain1 = ruleChainService.saveRuleChain(toRuleChain1);
RuleChainId toRuleChain1Id = toRuleChain1.getId();
RuleChainMetaData ruleChainMetaData = new RuleChainMetaData();
ruleChainMetaData.setRuleChainId(fromRuleChainId);
RuleNode toRuleChain1Node = new RuleNode();
toRuleChain1Node.setName("To Rule Chain 1");
toRuleChain1Node.setType(TB_RULE_CHAIN_INPUT_NODE);
toRuleChain1Node.setConfiguration(JacksonUtil.newObjectNode()
.put("ruleChainId", toRuleChain1Id.toString()));
RuleNode toRuleChain1Node2 = new RuleNode();
toRuleChain1Node2.setName("To Rule Chain 1");
toRuleChain1Node2.setType(TB_RULE_CHAIN_INPUT_NODE);
toRuleChain1Node2.setConfiguration(JacksonUtil.newObjectNode()
.put("ruleChainId", toRuleChain1Id.toString()));
List<RuleNode> ruleNodes = new ArrayList<>();
ruleNodes.add(toRuleChain1Node);
ruleNodes.add(toRuleChain1Node2);
ruleChainMetaData.setFirstNodeIndex(0);
ruleChainMetaData.setNodes(ruleNodes);
ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData, Function.identity());
List<EntityRelation> relations = relationService.findByFromAndType(tenantId, fromRuleChainId, USES_TYPE, RelationTypeGroup.COMMON);
assertThat(relations).singleElement().satisfies(relationToRuleChain1 -> {
assertThat(relationToRuleChain1.getFrom()).isEqualTo(fromRuleChainId);
assertThat(relationToRuleChain1.getTo()).isEqualTo(toRuleChain1Id);
});
RuleChain toRuleChain2 = new RuleChain();
toRuleChain2.setName("To Rule Chain 2");
toRuleChain2.setTenantId(tenantId);
toRuleChain2 = ruleChainService.saveRuleChain(toRuleChain2);
RuleChainId toRuleChain2Id = toRuleChain2.getId();
RuleNode toRuleChain2Node = new RuleNode();
toRuleChain2Node.setName("To Rule Chain 2");
toRuleChain2Node.setType(TB_RULE_CHAIN_INPUT_NODE);
toRuleChain2Node.setConfiguration(JacksonUtil.newObjectNode()
.put("ruleChainId", toRuleChain2Id.toString()));
List<RuleNode> newRuleNodes = new ArrayList<>();
newRuleNodes.add(toRuleChain2Node);
newRuleNodes.add(toRuleChain1Node);
ruleChainMetaData = ruleChainService.loadRuleChainMetaData(tenantId, ruleChainMetaData.getRuleChainId());
ruleChainMetaData.setNodes(newRuleNodes);
ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData, Function.identity());
List<EntityRelation> newRelations = relationService.findByFromAndType(tenantId, fromRuleChainId, USES_TYPE, RelationTypeGroup.COMMON);
assertThat(newRelations).hasSize(2);
assertThat(newRelations).anySatisfy(relationToRuleChain1 -> {
assertThat(relationToRuleChain1.getFrom()).isEqualTo(fromRuleChainId);
assertThat(relationToRuleChain1.getTo()).isEqualTo(toRuleChain1Id);
});
assertThat(newRelations).anySatisfy(relationToRuleChain2 -> {
assertThat(relationToRuleChain2.getFrom()).isEqualTo(fromRuleChainId);
assertThat(relationToRuleChain2.getTo()).isEqualTo(toRuleChain2Id);
});
}
private RuleChainId saveRuleChainAndSetAutoAssignToEdge(String name) {
RuleChain edgeRuleChain = new RuleChain();
edgeRuleChain.setTenantId(tenantId);
@ -395,9 +476,9 @@ public class RuleChainServiceTest extends AbstractServiceTest {
ruleChainMetaData.setFirstNodeIndex(0);
ruleChainMetaData.setNodes(ruleNodes);
ruleChainMetaData.addConnectionInfo(0,1,"success");
ruleChainMetaData.addConnectionInfo(0,2,"fail");
ruleChainMetaData.addConnectionInfo(1,2,"success");
ruleChainMetaData.addConnectionInfo(0, 1, "success");
ruleChainMetaData.addConnectionInfo(0, 2, "fail");
ruleChainMetaData.addConnectionInfo(1, 2, "success");
Assert.assertTrue(ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData, Function.identity()).isSuccess());
return ruleChainService.loadRuleChainMetaData(tenantId, ruleChainMetaData.getRuleChainId());
@ -434,10 +515,10 @@ public class RuleChainServiceTest extends AbstractServiceTest {
ruleChainMetaData.setFirstNodeIndex(0);
ruleChainMetaData.setNodes(ruleNodes);
ruleChainMetaData.addConnectionInfo(0,1,"success");
ruleChainMetaData.addConnectionInfo(0,2,"fail");
ruleChainMetaData.addConnectionInfo(1,2,"success");
ruleChainMetaData.addConnectionInfo(2,2,"success");
ruleChainMetaData.addConnectionInfo(0, 1, "success");
ruleChainMetaData.addConnectionInfo(0, 2, "fail");
ruleChainMetaData.addConnectionInfo(1, 2, "success");
ruleChainMetaData.addConnectionInfo(2, 2, "success");
return ruleChainMetaData;
}
@ -473,10 +554,10 @@ public class RuleChainServiceTest extends AbstractServiceTest {
ruleChainMetaData.setFirstNodeIndex(0);
ruleChainMetaData.setNodes(ruleNodes);
ruleChainMetaData.addConnectionInfo(0,1,"success");
ruleChainMetaData.addConnectionInfo(0,2,"fail");
ruleChainMetaData.addConnectionInfo(1,2,"success");
ruleChainMetaData.addConnectionInfo(2,0,"success");
ruleChainMetaData.addConnectionInfo(0, 1, "success");
ruleChainMetaData.addConnectionInfo(0, 2, "fail");
ruleChainMetaData.addConnectionInfo(1, 2, "success");
ruleChainMetaData.addConnectionInfo(2, 0, "success");
return ruleChainMetaData;
}
@ -582,16 +663,16 @@ public class RuleChainServiceTest extends AbstractServiceTest {
private RuleChain getRuleChain() {
String ruleChainStr = "{\n" +
" \"name\": \"Root Rule Chain\",\n" +
" \"type\": \"CORE\",\n" +
" \"firstRuleNodeId\": {\n" +
" \"entityType\": \"RULE_NODE\",\n" +
" \"id\": \"91ad0b00-e779-11ee-9cf0-15d8b6079fdb\"\n" +
" },\n" +
" \"debugMode\": false,\n" +
" \"configuration\": null,\n" +
" \"additionalInfo\": null\n" +
"}";
" \"name\": \"Root Rule Chain\",\n" +
" \"type\": \"CORE\",\n" +
" \"firstRuleNodeId\": {\n" +
" \"entityType\": \"RULE_NODE\",\n" +
" \"id\": \"91ad0b00-e779-11ee-9cf0-15d8b6079fdb\"\n" +
" },\n" +
" \"debugMode\": false,\n" +
" \"configuration\": null,\n" +
" \"additionalInfo\": null\n" +
"}";
return JacksonUtil.fromString(ruleChainStr, RuleChain.class);
}
}

3
dao/src/test/resources/application-test.properties

@ -108,6 +108,9 @@ cache.specs.qrCodeSettings.maxSize=10000
cache.specs.mobileSecretKey.timeToLiveInMinutes=1440
cache.specs.mobileSecretKey.maxSize=10000
cache.specs.trendzSettings.timeToLiveInMinutes=1440
cache.specs.trendzSettings.maxSize=10000
redis.connection.host=localhost
redis.connection.port=6379
redis.connection.db=0

6
edqs/src/main/resources/edqs.yml

@ -148,7 +148,11 @@ queue:
- key: max.poll.records
# Max poll records for edqs.state topic
value: "${TB_QUEUE_KAFKA_EDQS_STATE_MAX_POLL_RECORDS:512}"
# If you override any default Kafka topic name using environment variables, you must also specify the related consumer properties
# for the new topic in `consumer-properties-per-topic-inline`. Otherwise, the topic will not inherit its expected configuration (e.g., max.poll.records, timeouts, etc).
# Format: "topic1:key1=value1,key2=value2;topic2:key=value"
# Example: "tb_core_modified.notifications:max.poll.records=10;tb_edge_modified:max.poll.records=10,enable.auto.commit=true"
consumer-properties-per-topic-inline: "${TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE:}"
other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000"
other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside
# - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms

6
monitoring/src/main/java/org/thingsboard/monitoring/ThingsboardMonitoringApplication.java

@ -25,6 +25,7 @@ import org.springframework.context.event.EventListener;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.monitoring.service.BaseMonitoringService;
import org.thingsboard.monitoring.service.MonitoringEntityService;
import java.util.List;
import java.util.Map;
@ -38,6 +39,8 @@ public class ThingsboardMonitoringApplication {
@Autowired
private List<BaseMonitoringService<?, ?>> monitoringServices;
@Autowired
private MonitoringEntityService entityService;
@Value("${monitoring.monitoring_rate_ms}")
private int monitoringRateMs;
@ -50,6 +53,9 @@ public class ThingsboardMonitoringApplication {
@EventListener(ApplicationReadyEvent.class)
public void startMonitoring() {
entityService.checkEntities();
monitoringServices.forEach(BaseMonitoringService::init);
ScheduledExecutorService scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("monitoring-executor");
scheduler.scheduleWithFixedDelay(() -> {
monitoringServices.forEach(monitoringService -> {

9
monitoring/src/main/java/org/thingsboard/monitoring/client/TbClient.java

@ -15,17 +15,15 @@
*/
package org.thingsboard.monitoring.client;
import jakarta.annotation.PostConstruct;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.boot.web.client.RestTemplateBuilder;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import org.thingsboard.rest.client.RestClient;
import java.time.Duration;
@Component
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public class TbClient extends RestClient {
@Value("${monitoring.rest.username}")
@ -41,6 +39,11 @@ public class TbClient extends RestClient {
.build(), baseUrl);
}
@PostConstruct
private void init() {
logIn();
}
public String logIn() {
login(username, password);
return getToken();

69
monitoring/src/main/java/org/thingsboard/monitoring/client/WsClient.java

@ -36,8 +36,11 @@ import org.thingsboard.server.common.data.query.EntityListFilter;
import javax.net.ssl.SSLParameters;
import java.net.URI;
import java.nio.channels.NotYetConnectedException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@ -48,13 +51,13 @@ import java.util.stream.Collectors;
@Slf4j
public class WsClient extends WebSocketClient implements AutoCloseable {
public volatile JsonNode lastMsg;
public final List<JsonNode> lastMsgs = new ArrayList<>();
private CountDownLatch reply;
private CountDownLatch update;
private final Lock updateLock = new ReentrantLock();
private long requestTimeoutMs;
private final long requestTimeoutMs;
public WsClient(URI serverUri, long requestTimeoutMs) {
super(serverUri);
@ -63,7 +66,6 @@ public class WsClient extends WebSocketClient implements AutoCloseable {
@Override
public void onOpen(ServerHandshake serverHandshake) {
}
@Override
@ -73,8 +75,9 @@ public class WsClient extends WebSocketClient implements AutoCloseable {
}
updateLock.lock();
try {
lastMsg = JacksonUtil.toJsonNode(s);
log.trace("Received new msg: {}", lastMsg.toPrettyString());
JsonNode msg = JacksonUtil.toJsonNode(s);
lastMsgs.add(msg);
log.trace("Received new msg: {}", msg.toPrettyString());
if (update != null) {
update.countDown();
}
@ -96,11 +99,11 @@ public class WsClient extends WebSocketClient implements AutoCloseable {
log.error("WebSocket client error:", e);
}
public void registerWaitForUpdate() {
public void registerWaitForUpdates(int count) {
updateLock.lock();
try {
lastMsg = null;
update = new CountDownLatch(1);
lastMsgs.clear();
update = new CountDownLatch(count);
} finally {
updateLock.unlock();
}
@ -111,6 +114,7 @@ public class WsClient extends WebSocketClient implements AutoCloseable {
public void send(String text) throws NotYetConnectedException {
updateLock.lock();
try {
lastMsgs.clear();
reply = new CountDownLatch(1);
} finally {
updateLock.unlock();
@ -118,19 +122,19 @@ public class WsClient extends WebSocketClient implements AutoCloseable {
super.send(text);
}
public WsClient subscribeForTelemetry(List<UUID> devices, String key) {
public WsClient subscribeForTelemetry(List<UUID> devices, List<String> keys) {
EntityDataCmd cmd = new EntityDataCmd();
cmd.setCmdId(RandomUtils.nextInt(0, 1000));
EntityListFilter devicesFilter = new EntityListFilter();
devicesFilter.setEntityType(EntityType.DEVICE);
devicesFilter.setEntityList(devices.stream().map(UUID::toString).collect(Collectors.toList()));
EntityDataPageLink pageLink = new EntityDataPageLink(100,0, null, null);
EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, null);
EntityDataQuery devicesQuery = new EntityDataQuery(devicesFilter, pageLink, Collections.emptyList(), Collections.emptyList(), Collections.emptyList());
cmd.setQuery(devicesQuery);
LatestValueCmd latestCmd = new LatestValueCmd();
latestCmd.setKeys(List.of(new EntityKey(EntityKeyType.TIME_SERIES, key)));
latestCmd.setKeys(keys.stream().map(key -> new EntityKey(EntityKeyType.TIME_SERIES, key)).toList());
cmd.setLatestCmd(latestCmd);
CmdsWrapper wrapper = new CmdsWrapper();
@ -139,12 +143,12 @@ public class WsClient extends WebSocketClient implements AutoCloseable {
return this;
}
public JsonNode waitForUpdate(long ms) {
public List<JsonNode> waitForUpdates(long ms) {
log.trace("update latch count: {}", update.getCount());
try {
if (update.await(ms, TimeUnit.MILLISECONDS)) {
log.trace("Waited for update");
return getLastMsg();
return getLastMsgs();
}
} catch (InterruptedException e) {
log.debug("Failed to await reply", e);
@ -157,7 +161,8 @@ public class WsClient extends WebSocketClient implements AutoCloseable {
try {
if (reply.await(requestTimeoutMs, TimeUnit.MILLISECONDS)) {
log.trace("Waited for reply");
return getLastMsg();
List<JsonNode> lastMsgs = getLastMsgs();
return lastMsgs.isEmpty() ? null : lastMsgs.get(0);
}
} catch (InterruptedException e) {
log.debug("Failed to await reply", e);
@ -166,24 +171,30 @@ public class WsClient extends WebSocketClient implements AutoCloseable {
throw new IllegalStateException("No WS reply arrived within " + requestTimeoutMs + " ms");
}
private JsonNode getLastMsg() {
if (lastMsg != null) {
JsonNode errorMsg = lastMsg.get("errorMsg");
if (errorMsg != null && !errorMsg.isNull() && StringUtils.isNotEmpty(errorMsg.asText())) {
throw new RuntimeException("WS error from server: " + errorMsg.asText());
} else {
return lastMsg;
}
} else {
return null;
private List<JsonNode> getLastMsgs() {
if (lastMsgs.isEmpty()) {
return lastMsgs;
}
List<JsonNode> errors = lastMsgs.stream()
.map(msg -> msg.get("errorMsg"))
.filter(errorMsg -> errorMsg != null && !errorMsg.isNull() && StringUtils.isNotEmpty(errorMsg.asText()))
.toList();
if (!errors.isEmpty()) {
throw new RuntimeException("WS error from server: " + errors.stream()
.map(JsonNode::asText)
.collect(Collectors.joining(", ")));
}
return lastMsgs;
}
public Object getTelemetryUpdate(UUID deviceId, String key) {
JsonNode lastMsg = getLastMsg();
if (lastMsg == null || lastMsg.isNull()) return null;
EntityDataUpdate update = JacksonUtil.treeToValue(lastMsg, EntityDataUpdate.class);
return update.getLatest(deviceId, key);
public Map<String, String> getLatest(UUID deviceId) {
Map<String, String> updates = new HashMap<>();
getLastMsgs().forEach(msg -> {
EntityDataUpdate update = JacksonUtil.treeToValue(msg, EntityDataUpdate.class);
Map<String, String> latest = update.getLatest(deviceId);
updates.putAll(latest);
});
return updates;
}
@Override

11
monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportInfo.java

@ -20,16 +20,15 @@ import lombok.Data;
@Data
public class TransportInfo {
private final TransportType transportType;
private final String baseUrl;
private final String queue;
private final TransportType type;
private final TransportMonitoringTarget target;
@Override
public String toString() {
if (queue.equals("Main")) {
return String.format("*%s* (%s)", transportType.getName(), baseUrl);
if (target.getQueue().equals("Main")) {
return String.format("*%s* (%s)", type.getName(), target.getBaseUrl());
} else {
return String.format("*%s* (%s) _%s_", transportType.getName(), baseUrl, queue);
return String.format("*%s* (%s) _%s_", type.getName(), target.getBaseUrl(), target.getQueue());
}
}

6
monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringTarget.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.monitoring.config.transport;
import com.google.common.base.Strings;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import org.thingsboard.monitoring.config.MonitoringTarget;
@ -28,6 +29,7 @@ public class TransportMonitoringTarget implements MonitoringTarget {
private DeviceConfig device; // set manually during initialization
private String queue;
private boolean checkDomainIps;
private String namePrefix;
@Override
public UUID getDeviceId() {
@ -38,4 +40,8 @@ public class TransportMonitoringTarget implements MonitoringTarget {
return StringUtils.defaultIfEmpty(queue, "Main");
}
public String getNamePrefix() {
return Strings.nullToEmpty(namePrefix);
}
}

11
monitoring/src/main/java/org/thingsboard/monitoring/data/ServiceFailureException.java

@ -15,14 +15,21 @@
*/
package org.thingsboard.monitoring.data;
import lombok.Getter;
@Getter
public class ServiceFailureException extends RuntimeException {
public ServiceFailureException(Throwable cause) {
private final Object serviceKey;
public ServiceFailureException(Object serviceKey, Throwable cause) {
super(cause.getMessage(), cause);
this.serviceKey = serviceKey;
}
public ServiceFailureException(String message) {
public ServiceFailureException(Object serviceKey, String message) {
super(message);
this.serviceKey = serviceKey;
}
}

18
monitoring/src/main/java/org/thingsboard/monitoring/data/cmd/EntityDataUpdate.java

@ -19,9 +19,11 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.TsValue;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@Data
@ -31,14 +33,16 @@ public class EntityDataUpdate {
@JsonIgnoreProperties(ignoreUnknown = true)
private List<EntityData> update;
public String getLatest(UUID entityId, String key) {
if (update == null) return null;
return update.stream()
public Map<String, String> getLatest(UUID entityId) {
if (update == null || update.isEmpty()) {
return Collections.emptyMap();
}
Map<String, String> result = new HashMap<>();
update.stream()
.filter(entityData -> entityData.getEntityId().getId().equals(entityId)).findFirst()
.map(EntityData::getLatest).map(latest -> latest.get(EntityKeyType.TIME_SERIES))
.map(latest -> latest.get(key)).map(TsValue::getValue)
.orElse(null);
.ifPresent(latest -> latest.forEach((key, tsValue) -> result.put(key, tsValue.getValue())));
return result;
}
}

47
monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java

@ -22,7 +22,6 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.thingsboard.monitoring.client.TbClient;
import org.thingsboard.monitoring.client.WsClient;
import org.thingsboard.monitoring.config.MonitoringConfig;
import org.thingsboard.monitoring.config.MonitoringTarget;
@ -46,6 +45,8 @@ public abstract class BaseHealthChecker<C extends MonitoringConfig, T extends Mo
private Object info;
@Autowired
protected MonitoringEntityService entityService;
@Autowired
private MonitoringReporter reporter;
@Autowired
@ -57,18 +58,20 @@ public abstract class BaseHealthChecker<C extends MonitoringConfig, T extends Mo
private final Map<String, BaseHealthChecker<C, T>> associates = new HashMap<>();
public static final String TEST_TELEMETRY_KEY = "testData";
public static final String TEST_CF_TELEMETRY_KEY = "testDataCf";
@PostConstruct
private void init() {
info = getInfo();
}
protected abstract void initialize(TbClient tbClient);
protected abstract void initialize();
public final void check(WsClient wsClient) {
log.debug("[{}] Checking", info);
try {
wsClient.registerWaitForUpdate();
int expectedUpdatesCount = isCfMonitoringEnabled() ? 2 : 1;
wsClient.registerWaitForUpdates(expectedUpdatesCount);
String testValue = UUID.randomUUID().toString();
String testPayload = createTestPayload(testValue);
@ -79,16 +82,16 @@ public abstract class BaseHealthChecker<C extends MonitoringConfig, T extends Mo
reporter.reportLatency(Latencies.request(getKey()), stopWatch.getTime());
log.trace("[{}] Sent test payload ({})", info, testPayload);
} catch (Throwable e) {
throw new ServiceFailureException(e);
throw new ServiceFailureException(info, e);
}
log.trace("[{}] Waiting for WS update", info);
checkWsUpdate(wsClient, testValue);
checkWsUpdates(wsClient, testValue);
reporter.serviceIsOk(info);
reporter.serviceIsOk(MonitoredServiceKey.GENERAL);
} catch (ServiceFailureException serviceFailureException) {
reporter.serviceFailure(info, serviceFailureException);
} catch (ServiceFailureException e) {
reporter.serviceFailure(e.getServiceKey(), e);
} catch (Exception e) {
reporter.serviceFailure(MonitoredServiceKey.GENERAL, e);
}
@ -98,15 +101,26 @@ public abstract class BaseHealthChecker<C extends MonitoringConfig, T extends Mo
});
}
private void checkWsUpdate(WsClient wsClient, String testValue) {
private void checkWsUpdates(WsClient wsClient, String testValue) {
stopWatch.start();
wsClient.waitForUpdate(resultCheckTimeoutMs);
log.trace("[{}] Waited for WS update. Last WS msg: {}", info, wsClient.lastMsg);
Object update = wsClient.getTelemetryUpdate(target.getDeviceId(), TEST_TELEMETRY_KEY);
if (update == null) {
throw new ServiceFailureException("No WS update arrived within " + resultCheckTimeoutMs + " ms");
} else if (!update.toString().equals(testValue)) {
throw new ServiceFailureException("Was expecting value " + testValue + " but got " + update);
wsClient.waitForUpdates(resultCheckTimeoutMs);
log.trace("[{}] Waited for WS update. Last WS msgs: {}", info, wsClient.lastMsgs);
Map<String, String> latest = wsClient.getLatest(target.getDeviceId());
if (latest.isEmpty()) {
throw new ServiceFailureException(info, "No WS update arrived within " + resultCheckTimeoutMs + " ms");
}
String actualValue = latest.get(TEST_TELEMETRY_KEY);
if (!testValue.equals(actualValue)) {
throw new ServiceFailureException(info, "Was expecting value " + testValue + " but got " + actualValue);
}
if (isCfMonitoringEnabled()) {
String cfTestValue = testValue + "-cf";
String actualCfValue = latest.get(TEST_CF_TELEMETRY_KEY);
if (actualCfValue == null) {
throw new ServiceFailureException(info, "No calculated field value arrived");
} else if (!cfTestValue.equals(actualCfValue)) {
throw new ServiceFailureException(info, "Was expecting calculated field value " + cfTestValue + " but got " + actualCfValue);
}
}
reporter.reportLatency(Latencies.wsUpdate(getKey()), stopWatch.getTime());
}
@ -121,6 +135,9 @@ public abstract class BaseHealthChecker<C extends MonitoringConfig, T extends Mo
protected abstract void destroyClient() throws Exception;
protected abstract Object getInfo();
protected abstract String getKey();
protected abstract boolean isCfMonitoringEnabled();
}

47
monitoring/src/main/java/org/thingsboard/monitoring/service/BaseMonitoringService.java

@ -16,7 +16,6 @@
package org.thingsboard.monitoring.service;
import com.google.common.collect.Sets;
import jakarta.annotation.PostConstruct;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@ -31,7 +30,6 @@ import org.thingsboard.monitoring.config.MonitoringTarget;
import org.thingsboard.monitoring.data.Latencies;
import org.thingsboard.monitoring.data.MonitoredServiceKey;
import org.thingsboard.monitoring.data.ServiceFailureException;
import org.thingsboard.monitoring.service.transport.TransportHealthChecker;
import org.thingsboard.monitoring.util.TbStopWatch;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.page.PageData;
@ -59,6 +57,9 @@ import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.thingsboard.monitoring.service.BaseHealthChecker.TEST_CF_TELEMETRY_KEY;
import static org.thingsboard.monitoring.service.BaseHealthChecker.TEST_TELEMETRY_KEY;
@Slf4j
public abstract class BaseMonitoringService<C extends MonitoringConfig<T>, T extends MonitoringTarget> {
@ -79,14 +80,15 @@ public abstract class BaseMonitoringService<C extends MonitoringConfig<T>, T ext
protected ApplicationContext applicationContext;
@Value("${monitoring.edqs.enabled:false}")
private boolean edqsMonitoringEnabled;
private boolean checkEdqs;
@Value("${monitoring.calculated_fields.enabled:true}")
protected boolean checkCalculatedFields;
@PostConstruct
private void init() {
public void init() {
if (configs == null || configs.isEmpty()) {
return;
}
tbClient.logIn();
configs.forEach(config -> {
config.getTargets().forEach(target -> {
BaseHealthChecker<C, T> healthChecker = initHealthChecker(target, config);
@ -104,7 +106,7 @@ public abstract class BaseMonitoringService<C extends MonitoringConfig<T>, T ext
private BaseHealthChecker<C, T> initHealthChecker(T target, C config) {
BaseHealthChecker<C, T> healthChecker = (BaseHealthChecker<C, T>) createHealthChecker(config, target);
log.info("Initializing {} for {}", healthChecker.getClass().getSimpleName(), target.getBaseUrl());
healthChecker.initialize(tbClient);
healthChecker.initialize();
devices.add(target.getDeviceId());
return healthChecker;
}
@ -121,7 +123,7 @@ public abstract class BaseMonitoringService<C extends MonitoringConfig<T>, T ext
try (WsClient wsClient = wsClientFactory.createClient(accessToken)) {
stopWatch.start();
wsClient.subscribeForTelemetry(devices, TransportHealthChecker.TEST_TELEMETRY_KEY).waitForReply();
wsClient.subscribeForTelemetry(devices, getTestTelemetryKeys()).waitForReply();
reporter.reportLatency(Latencies.WS_SUBSCRIBE, stopWatch.getTime());
for (BaseHealthChecker<C, T> healthChecker : healthCheckers) {
@ -129,22 +131,17 @@ public abstract class BaseMonitoringService<C extends MonitoringConfig<T>, T ext
}
}
if (edqsMonitoringEnabled) {
try {
stopWatch.start();
checkEdqs();
reporter.reportLatency(Latencies.EDQS_QUERY, stopWatch.getTime());
reporter.serviceIsOk(MonitoredServiceKey.EDQS);
} catch (ServiceFailureException e) {
reporter.serviceFailure(MonitoredServiceKey.EDQS, e);
} catch (Exception e) {
reporter.serviceFailure(MonitoredServiceKey.GENERAL, e);
}
if (checkEdqs) {
stopWatch.start();
checkEdqs();
reporter.reportLatency(Latencies.EDQS_QUERY, stopWatch.getTime());
reporter.serviceIsOk(MonitoredServiceKey.EDQS);
}
reporter.reportLatencies(tbClient);
reporter.reportLatencies();
log.debug("Finished {}", getName());
} catch (ServiceFailureException e) {
reporter.serviceFailure(e.getServiceKey(), e);
} catch (Throwable error) {
try {
reporter.serviceFailure(MonitoredServiceKey.GENERAL, error);
@ -199,7 +196,7 @@ public abstract class BaseMonitoringService<C extends MonitoringConfig<T>, T ext
.collect(Collectors.toSet());
Set<UUID> missing = Sets.difference(new HashSet<>(this.devices), devices);
if (!missing.isEmpty()) {
throw new ServiceFailureException("Missing devices in the response: " + missing);
throw new ServiceFailureException(MonitoredServiceKey.EDQS, "Missing devices in the response: " + missing);
}
result.getData().stream()
@ -211,7 +208,7 @@ public abstract class BaseMonitoringService<C extends MonitoringConfig<T>, T ext
Stream.of("name", "type", "testData").forEach(key -> {
TsValue value = values.get(key);
if (value == null || StringUtils.isBlank(value.getValue())) {
throw new ServiceFailureException("Missing " + key + " for device " + entityData.getEntityId());
throw new ServiceFailureException(MonitoredServiceKey.EDQS, "Missing " + key + " for device " + entityData.getEntityId());
}
});
});
@ -232,6 +229,10 @@ public abstract class BaseMonitoringService<C extends MonitoringConfig<T>, T ext
.collect(Collectors.toSet());
}
private List<String> getTestTelemetryKeys() {
return checkCalculatedFields ? List.of(TEST_TELEMETRY_KEY, TEST_CF_TELEMETRY_KEY) : List.of(TEST_TELEMETRY_KEY);
}
private void stopHealthChecker(BaseHealthChecker<C, T> healthChecker) throws Exception {
healthChecker.destroyClient();
devices.remove(healthChecker.getTarget().getDeviceId());

252
monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringEntityService.java

@ -0,0 +1,252 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.monitoring.service;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomStringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.RegexUtils;
import org.thingsboard.monitoring.client.TbClient;
import org.thingsboard.monitoring.config.transport.DeviceConfig;
import org.thingsboard.monitoring.config.transport.TransportMonitoringConfig;
import org.thingsboard.monitoring.config.transport.TransportMonitoringTarget;
import org.thingsboard.monitoring.config.transport.TransportType;
import org.thingsboard.monitoring.util.ResourceUtils;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.DeviceProfileType;
import org.thingsboard.server.common.data.DeviceTransportType;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Argument;
import org.thingsboard.server.common.data.cf.configuration.ArgumentType;
import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.common.data.cf.configuration.OutputType;
import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey;
import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration;
import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MBootstrapClientCredentials;
import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MDeviceCredentials;
import org.thingsboard.server.common.data.device.credentials.lwm2m.NoSecBootstrapClientCredential;
import org.thingsboard.server.common.data.device.credentials.lwm2m.NoSecClientCredential;
import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration;
import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration;
import org.thingsboard.server.common.data.device.data.DeviceData;
import org.thingsboard.server.common.data.device.data.Lwm2mDeviceTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration;
import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.DeviceProfileData;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.DeviceCredentialsType;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static org.thingsboard.monitoring.service.BaseHealthChecker.TEST_CF_TELEMETRY_KEY;
import static org.thingsboard.monitoring.service.BaseHealthChecker.TEST_TELEMETRY_KEY;
@Service
@Slf4j
@RequiredArgsConstructor
public class MonitoringEntityService {
private final TbClient tbClient;
@Value("${monitoring.calculated_fields.enabled:true}")
private boolean calculatedFieldsMonitoringEnabled;
public void checkEntities() {
RuleChain ruleChain = tbClient.getRuleChains(RuleChainType.CORE, new PageLink(10)).getData().stream()
.filter(RuleChain::isRoot)
.findFirst().orElseThrow();
RuleChainId ruleChainId = ruleChain.getId();
JsonNode ruleChainDescriptor = ResourceUtils.getResource("rule_chain.json");
List<String> attributeKeys = tbClient.getAttributeKeys(ruleChainId);
Map<String, String> attributes = tbClient.getAttributeKvEntries(ruleChainId, attributeKeys).stream()
.collect(Collectors.toMap(KvEntry::getKey, KvEntry::getValueAsString));
int currentVersion = Integer.parseInt(attributes.getOrDefault("version", "0"));
int newVersion = ruleChainDescriptor.get("version").asInt();
if (currentVersion == newVersion) {
log.info("Not updating rule chain, version is the same ({})", currentVersion);
return;
} else {
log.info("Updating rule chain '{}' from version {} to {}", ruleChain.getName(), currentVersion, newVersion);
}
String metadataJson = RegexUtils.replace(ruleChainDescriptor.get("metadata").toString(),
"\\$\\{MONITORING:(.+?)}", matchResult -> {
String key = matchResult.group(1);
String value = attributes.get(key);
if (value == null) {
throw new IllegalArgumentException("No attribute found for key " + key);
}
log.info("Using {}: {}", key, value);
return value;
});
RuleChainMetaData metaData = JacksonUtil.fromString(metadataJson, RuleChainMetaData.class);
metaData.setRuleChainId(ruleChainId);
tbClient.saveRuleChainMetaData(metaData);
tbClient.saveEntityAttributesV2(ruleChainId, DataConstants.SERVER_SCOPE, JacksonUtil.newObjectNode()
.put("version", newVersion));
}
public Asset getOrCreateMonitoringAsset() {
String assetName = "[Monitoring] Latencies";
return tbClient.findAsset(assetName).orElseGet(() -> {
Asset asset = new Asset();
asset.setType("Monitoring");
asset.setName(assetName);
asset = tbClient.saveAsset(asset);
log.info("Created monitoring asset {}", asset.getId());
return asset;
});
}
public void checkEntities(TransportMonitoringConfig config, TransportMonitoringTarget target) {
Device device = getOrCreateDevice(config, target);
DeviceCredentials credentials = tbClient.getDeviceCredentialsByDeviceId(device.getId())
.orElseThrow(() -> new IllegalArgumentException("No credentials found for device " + device.getId()));
DeviceConfig deviceConfig = new DeviceConfig();
deviceConfig.setId(device.getId().toString());
deviceConfig.setName(device.getName());
deviceConfig.setCredentials(credentials);
target.setDevice(deviceConfig);
}
private Device getOrCreateDevice(TransportMonitoringConfig config, TransportMonitoringTarget target) {
TransportType transportType = config.getTransportType();
String deviceName = String.format("%s %s (%s) - %s", target.getNamePrefix(), transportType.getName(), target.getQueue(), target.getBaseUrl()).trim();
Device device = tbClient.getTenantDevice(deviceName).orElse(null);
if (device != null) {
if (calculatedFieldsMonitoringEnabled) {
CalculatedField calculatedField = tbClient.getCalculatedFieldsByEntityId(device.getId(), new PageLink(1, 0, TEST_CF_TELEMETRY_KEY))
.getData().stream().findFirst().orElse(null);
if (calculatedField == null) {
createCalculatedField(device);
}
}
return device;
}
log.info("Creating new device '{}'", deviceName);
device = new Device();
device.setName(deviceName);
DeviceCredentials credentials = new DeviceCredentials();
credentials.setCredentialsId(RandomStringUtils.randomAlphabetic(20));
DeviceData deviceData = new DeviceData();
deviceData.setConfiguration(new DefaultDeviceConfiguration());
DeviceProfile deviceProfile = getOrCreateDeviceProfile(config, target);
device.setType(deviceProfile.getName());
device.setDeviceProfileId(deviceProfile.getId());
if (transportType != TransportType.LWM2M) {
deviceData.setTransportConfiguration(new DefaultDeviceTransportConfiguration());
credentials.setCredentialsType(DeviceCredentialsType.ACCESS_TOKEN);
} else {
deviceData.setTransportConfiguration(new Lwm2mDeviceTransportConfiguration());
credentials.setCredentialsType(DeviceCredentialsType.LWM2M_CREDENTIALS);
LwM2MDeviceCredentials lwm2mCreds = new LwM2MDeviceCredentials();
NoSecClientCredential client = new NoSecClientCredential();
client.setEndpoint(credentials.getCredentialsId());
lwm2mCreds.setClient(client);
LwM2MBootstrapClientCredentials bootstrap = new LwM2MBootstrapClientCredentials();
bootstrap.setBootstrapServer(new NoSecBootstrapClientCredential());
bootstrap.setLwm2mServer(new NoSecBootstrapClientCredential());
lwm2mCreds.setBootstrap(bootstrap);
credentials.setCredentialsValue(JacksonUtil.toString(lwm2mCreds));
}
device = tbClient.saveDeviceWithCredentials(device, credentials).get();
if (calculatedFieldsMonitoringEnabled) {
createCalculatedField(device);
}
return device;
}
private DeviceProfile getOrCreateDeviceProfile(TransportMonitoringConfig config, TransportMonitoringTarget target) {
TransportType transportType = config.getTransportType();
String profileName = String.format("%s %s (%s)", target.getNamePrefix(), transportType.getName(), target.getQueue()).trim();
DeviceProfile deviceProfile = tbClient.getDeviceProfiles(new PageLink(1, 0, profileName)).getData()
.stream().findFirst().orElse(null);
if (deviceProfile != null) {
return deviceProfile;
}
log.info("Creating new device profile '{}'", profileName);
if (transportType != TransportType.LWM2M) {
deviceProfile = new DeviceProfile();
deviceProfile.setType(DeviceProfileType.DEFAULT);
deviceProfile.setTransportType(DeviceTransportType.DEFAULT);
DeviceProfileData profileData = new DeviceProfileData();
profileData.setConfiguration(new DefaultDeviceProfileConfiguration());
profileData.setTransportConfiguration(new DefaultDeviceProfileTransportConfiguration());
deviceProfile.setProfileData(profileData);
} else {
tbClient.getResources(new PageLink(1, 0, "LwM2M Monitoring")).getData()
.stream().findFirst()
.orElseGet(() -> {
TbResource newResource = ResourceUtils.getResource("lwm2m/resource.json", TbResource.class);
log.info("Creating LwM2M resource");
return tbClient.saveResource(newResource);
});
deviceProfile = ResourceUtils.getResource("lwm2m/device_profile.json", DeviceProfile.class);
}
deviceProfile.setName(profileName);
deviceProfile.setDefaultQueueName(target.getQueue());
return tbClient.saveDeviceProfile(deviceProfile);
}
private void createCalculatedField(Device device) {
log.info("Creating calculated field for device '{}'", device.getName());
CalculatedField calculatedField = new CalculatedField();
calculatedField.setName(TEST_CF_TELEMETRY_KEY);
calculatedField.setEntityId(device.getId());
calculatedField.setType(CalculatedFieldType.SCRIPT);
ScriptCalculatedFieldConfiguration configuration = new ScriptCalculatedFieldConfiguration();
Argument testDataArgument = new Argument();
testDataArgument.setRefEntityKey(new ReferencedEntityKey(TEST_TELEMETRY_KEY, ArgumentType.TS_LATEST, null));
configuration.setArguments(Map.of(
TEST_TELEMETRY_KEY, testDataArgument
));
configuration.setExpression("return { \"" + TEST_CF_TELEMETRY_KEY + "\": " + TEST_TELEMETRY_KEY + " + \"-cf\" };");
Output output = new Output();
output.setType(OutputType.TIME_SERIES);
configuration.setOutput(output);
calculatedField.setConfiguration(configuration);
calculatedField.setDebugMode(true);
tbClient.saveCalculatedField(calculatedField);
}
}

16
monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringReporter.java

@ -46,6 +46,8 @@ import java.util.stream.Collectors;
public class MonitoringReporter {
private final NotificationService notificationService;
private final TbClient tbClient;
private final MonitoringEntityService entityService;
private final Map<String, Latency> latencies = new ConcurrentHashMap<>();
private final Map<Object, AtomicInteger> failuresCounters = new ConcurrentHashMap<>();
@ -62,7 +64,7 @@ public class MonitoringReporter {
@Value("${monitoring.latency.reporting_asset_id}")
private String reportingAssetId;
public void reportLatencies(TbClient tbClient) {
public void reportLatencies() {
if (latencies.isEmpty()) {
return;
}
@ -81,15 +83,7 @@ public class MonitoringReporter {
try {
if (StringUtils.isBlank(reportingAssetId)) {
String assetName = "[Monitoring] Latencies";
Asset monitoringAsset = tbClient.findAsset(assetName).orElseGet(() -> {
Asset asset = new Asset();
asset.setType("Monitoring");
asset.setName(assetName);
asset = tbClient.saveAsset(asset);
log.info("Created monitoring asset {}", asset.getId());
return asset;
});
Asset monitoringAsset = entityService.getOrCreateMonitoringAsset();
reportingAssetId = monitoringAsset.getId().toString();
}
@ -113,7 +107,7 @@ public class MonitoringReporter {
public void serviceFailure(Object serviceKey, Throwable error) {
if (log.isDebugEnabled()) {
log.error("Error occurred", error);
log.error("[{}] Error occurred", serviceKey, error);
}
int failuresCount = failuresCounters.computeIfAbsent(serviceKey, k -> new AtomicInteger()).incrementAndGet();
ServiceFailureNotification notification = new ServiceFailureNotification(serviceKey, error, failuresCount);

118
monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java

@ -17,54 +17,27 @@ package org.thingsboard.monitoring.service.transport;
import com.fasterxml.jackson.databind.node.TextNode;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomStringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.monitoring.client.TbClient;
import org.thingsboard.monitoring.config.transport.DeviceConfig;
import org.thingsboard.monitoring.config.transport.TransportInfo;
import org.thingsboard.monitoring.config.transport.TransportMonitoringConfig;
import org.thingsboard.monitoring.config.transport.TransportMonitoringTarget;
import org.thingsboard.monitoring.config.transport.TransportType;
import org.thingsboard.monitoring.service.BaseHealthChecker;
import org.thingsboard.monitoring.util.ResourceUtils;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.DeviceProfileType;
import org.thingsboard.server.common.data.DeviceTransportType;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MBootstrapClientCredentials;
import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MDeviceCredentials;
import org.thingsboard.server.common.data.device.credentials.lwm2m.NoSecBootstrapClientCredential;
import org.thingsboard.server.common.data.device.credentials.lwm2m.NoSecClientCredential;
import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration;
import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration;
import org.thingsboard.server.common.data.device.data.DeviceData;
import org.thingsboard.server.common.data.device.data.Lwm2mDeviceTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration;
import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.DeviceProfileData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.DeviceCredentialsType;
@Slf4j
public abstract class TransportHealthChecker<C extends TransportMonitoringConfig> extends BaseHealthChecker<C, TransportMonitoringTarget> {
@Value("${monitoring.calculated_fields.enabled:true}")
private boolean calculatedFieldsMonitoringEnabled;
public TransportHealthChecker(C config, TransportMonitoringTarget target) {
super(config, target);
}
@Override
protected void initialize(TbClient tbClient) {
Device device = getOrCreateDevice(tbClient);
DeviceCredentials credentials = tbClient.getDeviceCredentialsByDeviceId(device.getId())
.orElseThrow(() -> new IllegalArgumentException("No credentials found for device " + device.getId()));
DeviceConfig deviceConfig = new DeviceConfig();
deviceConfig.setId(device.getId().toString());
deviceConfig.setName(device.getName());
deviceConfig.setCredentials(credentials);
target.setDevice(deviceConfig);
protected void initialize() {
entityService.checkEntities(config, target);
}
@Override
@ -74,7 +47,7 @@ public abstract class TransportHealthChecker<C extends TransportMonitoringConfig
@Override
protected Object getInfo() {
return new TransportInfo(getTransportType(), target.getBaseUrl(), target.getQueue());
return new TransportInfo(getTransportType(), target);
}
@Override
@ -84,80 +57,9 @@ public abstract class TransportHealthChecker<C extends TransportMonitoringConfig
protected abstract TransportType getTransportType();
private Device getOrCreateDevice(TbClient tbClient) {
TransportType transportType = config.getTransportType();
String deviceName = String.format("%s (%s) - %s", transportType.getName(), target.getQueue(), target.getBaseUrl());
Device device = tbClient.getTenantDevice(deviceName).orElse(null);
if (device != null) {
return device;
}
log.info("Creating new device '{}'", deviceName);
device = new Device();
device.setName(deviceName);
DeviceCredentials credentials = new DeviceCredentials();
credentials.setCredentialsId(RandomStringUtils.randomAlphabetic(20));
DeviceData deviceData = new DeviceData();
deviceData.setConfiguration(new DefaultDeviceConfiguration());
DeviceProfile deviceProfile = getOrCreateDeviceProfile(tbClient);
device.setType(deviceProfile.getName());
device.setDeviceProfileId(deviceProfile.getId());
if (transportType != TransportType.LWM2M) {
deviceData.setTransportConfiguration(new DefaultDeviceTransportConfiguration());
credentials.setCredentialsType(DeviceCredentialsType.ACCESS_TOKEN);
} else {
deviceData.setTransportConfiguration(new Lwm2mDeviceTransportConfiguration());
credentials.setCredentialsType(DeviceCredentialsType.LWM2M_CREDENTIALS);
LwM2MDeviceCredentials lwm2mCreds = new LwM2MDeviceCredentials();
NoSecClientCredential client = new NoSecClientCredential();
client.setEndpoint(credentials.getCredentialsId());
lwm2mCreds.setClient(client);
LwM2MBootstrapClientCredentials bootstrap = new LwM2MBootstrapClientCredentials();
bootstrap.setBootstrapServer(new NoSecBootstrapClientCredential());
bootstrap.setLwm2mServer(new NoSecBootstrapClientCredential());
lwm2mCreds.setBootstrap(bootstrap);
credentials.setCredentialsValue(JacksonUtil.toString(lwm2mCreds));
}
return tbClient.saveDeviceWithCredentials(device, credentials).get();
}
private DeviceProfile getOrCreateDeviceProfile(TbClient tbClient) {
TransportType transportType = config.getTransportType();
String profileName = String.format("%s (%s)", transportType.getName(), target.getQueue());
DeviceProfile deviceProfile = tbClient.getDeviceProfiles(new PageLink(1, 0, profileName)).getData()
.stream().findFirst().orElse(null);
if (deviceProfile != null) {
return deviceProfile;
}
log.info("Creating new device profile '{}'", profileName);
if (transportType != TransportType.LWM2M) {
deviceProfile = new DeviceProfile();
deviceProfile.setType(DeviceProfileType.DEFAULT);
deviceProfile.setTransportType(DeviceTransportType.DEFAULT);
DeviceProfileData profileData = new DeviceProfileData();
profileData.setConfiguration(new DefaultDeviceProfileConfiguration());
profileData.setTransportConfiguration(new DefaultDeviceProfileTransportConfiguration());
deviceProfile.setProfileData(profileData);
} else {
tbClient.getResources(new PageLink(1, 0, "LwM2M Monitoring")).getData()
.stream().findFirst()
.orElseGet(() -> {
TbResource newResource = ResourceUtils.getResource("lwm2m/resource.json", TbResource.class);
log.info("Creating LwM2M resource");
return tbClient.saveResource(newResource);
});
deviceProfile = ResourceUtils.getResource("lwm2m/device_profile.json", DeviceProfile.class);
}
deviceProfile.setName(profileName);
deviceProfile.setDefaultQueueName(target.getQueue());
return tbClient.saveDeviceProfile(deviceProfile);
@Override
protected boolean isCfMonitoringEnabled() {
return calculatedFieldsMonitoringEnabled;
}
}

16
monitoring/src/main/java/org/thingsboard/monitoring/util/ResourceUtils.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.monitoring.util;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.SneakyThrows;
import org.thingsboard.common.util.JacksonUtil;
@ -24,14 +25,21 @@ public class ResourceUtils {
@SneakyThrows
public static <T> T getResource(String path, Class<T> type) {
InputStream resource = ResourceUtils.class.getClassLoader().getResourceAsStream(path);
if (resource == null) {
throw new IllegalArgumentException("Resource not found for path " + path);
}
InputStream resource = getResourceStream(path);
return JacksonUtil.OBJECT_MAPPER.readValue(resource, type);
}
@SneakyThrows
public static JsonNode getResource(String path) {
InputStream resource = getResourceStream(path);
return JacksonUtil.OBJECT_MAPPER.readTree(resource);
}
public static InputStream getResourceAsStream(String path) {
return getResourceStream(path);
}
private static InputStream getResourceStream(String path) {
InputStream resource = ResourceUtils.class.getClassLoader().getResourceAsStream(path);
if (resource == null) {
throw new IllegalArgumentException("Resource not found for path " + path);

327
monitoring/src/main/resources/root_rule_chain.json → monitoring/src/main/resources/rule_chain.json

@ -1,257 +1,232 @@
{
"version": 1,
"ruleChain": {
"additionalInfo": null,
"name": "Root Rule Chain",
"type": "CORE",
"firstRuleNodeId": null,
"root": false,
"debugMode": false,
"configuration": null,
"externalId": null
"additionalInfo": null
},
"metadata": {
"firstNodeIndex": 12,
"firstNodeIndex": 9,
"nodes": [
{
"additionalInfo": {
"description": null,
"layoutX": 1202,
"layoutY": 221
},
"type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode",
"name": "Save Timeseries",
"debugSettings": {
"failuresEnabled": true,
"allEnabled": false,
"allEnabledUntil": 1735310701003
},
"singletonMode": false,
"queueName": null,
"configurationVersion": 1,
"configuration": {
"defaultTTL": 0,
"useServerTs": false,
"processingSettings": {
"type": "ON_EVERY_MESSAGE"
}
},
"externalId": null
"additionalInfo": {
"description": null,
"layoutX": 1202,
"layoutY": 221
}
},
{
"additionalInfo": {
"layoutX": 1000,
"layoutY": 167
},
"type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode",
"name": "Save Attributes",
"debugSettings": null,
"singletonMode": false,
"queueName": null,
"configurationVersion": 3,
"configuration": {
"scope": "CLIENT_SCOPE",
"notifyDevice": false,
"processingSettings": {
"type": "ON_EVERY_MESSAGE"
},
"scope": "CLIENT_SCOPE",
"notifyDevice": false,
"sendAttributesUpdatedNotification": false,
"updateAttributesOnlyOnValueChange": false
},
"externalId": null
"additionalInfo": {
"layoutX": 1000,
"layoutY": 167
}
},
{
"additionalInfo": {
"layoutX": 566,
"layoutY": 302
},
"type": "org.thingsboard.rule.engine.filter.TbMsgTypeSwitchNode",
"name": "Message Type Switch",
"debugSettings": null,
"singletonMode": false,
"queueName": null,
"configurationVersion": 0,
"configuration": {
"version": 0
},
"externalId": null
"additionalInfo": {
"layoutX": 566,
"layoutY": 302
}
},
{
"additionalInfo": {
"layoutX": 1000,
"layoutY": 381
},
"type": "org.thingsboard.rule.engine.action.TbLogNode",
"name": "Log RPC from Device",
"debugSettings": null,
"singletonMode": false,
"queueName": null,
"configurationVersion": 0,
"configuration": {
"scriptLang": "TBEL",
"jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);",
"tbelScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);"
},
"externalId": null
},
{
"additionalInfo": {
"layoutX": 1000,
"layoutY": 494
},
"layoutY": 381
}
},
{
"type": "org.thingsboard.rule.engine.action.TbLogNode",
"name": "Log Other",
"debugSettings": null,
"singletonMode": false,
"queueName": null,
"configurationVersion": 0,
"configuration": {
"scriptLang": "TBEL",
"jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);",
"tbelScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);"
},
"externalId": null
},
{
"additionalInfo": {
"layoutX": 1000,
"layoutY": 583
},
"layoutY": 494
}
},
{
"type": "org.thingsboard.rule.engine.rpc.TbSendRPCRequestNode",
"name": "RPC Call Request",
"debugSettings": null,
"singletonMode": false,
"queueName": null,
"configurationVersion": 0,
"configuration": {
"timeoutInSeconds": 60
},
"externalId": null
},
{
"additionalInfo": {
"layoutX": 255,
"layoutY": 301
},
"type": "org.thingsboard.rule.engine.filter.TbOriginatorTypeFilterNode",
"name": "Is Entity Group",
"singletonMode": false,
"configurationVersion": 0,
"configuration": {
"originatorTypes": [
"ENTITY_GROUP"
]
},
"externalId": null
},
{
"additionalInfo": {
"layoutX": 319,
"layoutY": 109
},
"type": "org.thingsboard.rule.engine.filter.TbMsgTypeFilterNode",
"name": "Post attributes or RPC request",
"singletonMode": false,
"configurationVersion": 0,
"configuration": {
"messageTypes": [
"POST_ATTRIBUTES_REQUEST",
"RPC_CALL_FROM_SERVER_TO_DEVICE"
]
},
"externalId": null
"layoutX": 1000,
"layoutY": 583
}
},
{
"additionalInfo": {
"layoutX": 627,
"layoutY": 108
"type": "org.thingsboard.rule.engine.profile.TbDeviceProfileNode",
"name": "Device Profile Node",
"debugSettings": {
"failuresEnabled": true,
"allEnabled": false,
"allEnabledUntil": 1735310701003
},
"type": "org.thingsboard.rule.engine.transform.TbDuplicateMsgToGroupNode",
"name": "Duplicate To Group Entities",
"singletonMode": false,
"configurationVersion": 0,
"queueName": null,
"configurationVersion": 1,
"configuration": {
"entityGroupId": null,
"entityGroupIsMessageOriginator": true
"persistAlarmRulesState": false,
"fetchAlarmRulesStateOnStart": false
},
"externalId": null
},
{
"additionalInfo": {
"description": "Process incoming messages from devices with the alarm rules defined in the device profile. Dispatch all incoming messages with \"Success\" relation type.",
"layoutX": 45,
"layoutY": 359
},
"type": "org.thingsboard.rule.engine.profile.TbDeviceProfileNode",
"name": "Device Profile Node",
"singletonMode": false,
"configurationVersion": 0,
"configuration": {
"persistAlarmRulesState": false,
"fetchAlarmRulesStateOnStart": false
},
"externalId": null
}
},
{
"additionalInfo": {
"description": "",
"layoutX": 160,
"layoutY": 631
},
"type": "org.thingsboard.rule.engine.filter.TbJsFilterNode",
"name": "Test JS script",
"debugSettings": null,
"singletonMode": false,
"queueName": null,
"configurationVersion": 0,
"configuration": {
"scriptLang": "JS",
"jsScript": "var test = {\n a: 'a',\n b: 'b'\n};\nreturn test.a === 'a' && test.b === 'b';",
"jsScript": "var test = {\n a: 'a',\n b: 'b'\n};\n\nreturn test.a === 'a' && test.b === 'b';",
"tbelScript": "return msg.temperature > 20;"
},
"externalId": null
},
{
"additionalInfo": {
"description": "",
"layoutX": 427,
"layoutY": 541
},
"layoutX": 251,
"layoutY": 499
}
},
{
"type": "org.thingsboard.rule.engine.filter.TbJsFilterNode",
"name": "Test TBEL script",
"debugSettings": null,
"singletonMode": false,
"queueName": null,
"configurationVersion": 0,
"configuration": {
"scriptLang": "TBEL",
"jsScript": "return msg.temperature > 20;",
"tbelScript": "var a = \"a\";\nvar b = \"b\";\nreturn a.equals(\"a\") && b.equals(\"b\");"
},
"externalId": null
},
{
"additionalInfo": {
"description": "",
"layoutX": 40,
"layoutY": 252
},
"layoutX": 317,
"layoutY": 355
}
},
{
"type": "org.thingsboard.rule.engine.transform.TbTransformMsgNode",
"name": "Add arrival timestamp",
"debugSettings": {
"failuresEnabled": true,
"allEnabled": false,
"allEnabledUntil": 1744642101587
},
"singletonMode": false,
"queueName": null,
"configurationVersion": 0,
"configuration": {
"scriptLang": "TBEL",
"jsScript": "return {msg: msg, metadata: metadata, msgType: msgType};",
"tbelScript": "metadata.arrivalTs = Date.now();\nreturn {msg: msg, metadata: metadata, msgType: msgType};"
},
"externalId": null
},
{
"additionalInfo": {
"description": "",
"layoutX": 1467,
"layoutY": 267
},
"layoutX": 40,
"layoutY": 252
}
},
{
"type": "org.thingsboard.rule.engine.transform.TbTransformMsgNode",
"name": "Calculate additional latencies",
"debugSettings": {
"failuresEnabled": true,
"allEnabled": false,
"allEnabledUntil": 1735310701003
},
"singletonMode": false,
"queueName": null,
"configurationVersion": 0,
"configuration": {
"scriptLang": "TBEL",
"jsScript": "return {msg: msg, metadata: metadata, msgType: msgType};",
"tbelScript": "var arrivalLatency = metadata.arrivalTs - metadata.ts;\nvar processingTime = Date.now() - metadata.arrivalTs;\nmsg = {\n arrivalLatency: arrivalLatency,\n processingTime: processingTime\n};\nreturn {msg: msg, metadata: metadata, msgType: msgType};"
},
"externalId": null
},
{
"additionalInfo": {
"description": "",
"layoutX": 1438,
"layoutY": 403
},
"layoutX": 1467,
"layoutY": 267
}
},
{
"type": "org.thingsboard.rule.engine.transform.TbChangeOriginatorNode",
"name": "To latencies asset",
"debugSettings": null,
"singletonMode": false,
"queueName": null,
"configurationVersion": 0,
"configuration": {
"originatorSource": "ENTITY",
@ -269,64 +244,79 @@
"fetchLastLevelOnly": false
}
},
"externalId": null
"additionalInfo": {
"description": "",
"layoutX": 1438,
"layoutY": 403
}
},
{
"additionalInfo": {
"description": null,
"layoutX": 1458,
"layoutY": 505
},
"type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode",
"name": "Save Timeseries",
"debugSettings": {
"failuresEnabled": true,
"allEnabled": false,
"allEnabledUntil": 1735310701003
},
"singletonMode": false,
"queueName": null,
"configurationVersion": 1,
"configuration": {
"defaultTTL": 0,
"useServerTs": false,
"processingSettings": {
"type": "ON_EVERY_MESSAGE"
}
},
"externalId": null
"additionalInfo": {
"description": null,
"layoutX": 1458,
"layoutY": 505
}
},
{
"additionalInfo": {
"description": "",
"layoutX": 928,
"layoutY": 266
},
"type": "org.thingsboard.rule.engine.filter.TbCheckMessageNode",
"name": "Has testData",
"debugSettings": null,
"singletonMode": false,
"queueName": null,
"configurationVersion": 0,
"configuration": {
"messageNames": [
"testData"
"testData",
"testDataCf"
],
"metadataNames": [],
"checkAllKeys": true
"checkAllKeys": false
},
"externalId": null
"additionalInfo": {
"description": "",
"layoutX": 928,
"layoutY": 266
}
},
{
"additionalInfo": {
"description": null,
"layoutX": 1203,
"layoutY": 327
},
"type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode",
"name": "Save Timeseries with TTL",
"debugSettings": {
"failuresEnabled": true,
"allEnabled": false,
"allEnabledUntil": 1742305233839
},
"singletonMode": false,
"queueName": null,
"configurationVersion": 1,
"configuration": {
"defaultTTL": 180,
"useServerTs": false,
"processingSettings": {
"type": "ON_EVERY_MESSAGE"
}
},
"defaultTTL": 60,
"useServerTs": null
},
"externalId": null
"additionalInfo": {
"description": "",
"layoutX": 1203,
"layoutY": 327
}
}
],
"connections": [
@ -352,23 +342,13 @@
},
{
"fromIndex": 2,
"toIndex": 16,
"toIndex": 13,
"type": "Post telemetry"
},
{
"fromIndex": 6,
"toIndex": 2,
"type": "False"
},
{
"fromIndex": 6,
"toIndex": 7,
"type": "True"
},
{
"fromIndex": 7,
"toIndex": 2,
"type": "False"
"type": "Success"
},
{
"fromIndex": 7,
@ -378,54 +358,39 @@
{
"fromIndex": 8,
"toIndex": 2,
"type": "Success"
"type": "True"
},
{
"fromIndex": 9,
"toIndex": 10,
"toIndex": 6,
"type": "Success"
},
{
"fromIndex": 10,
"toIndex": 11,
"type": "True"
"type": "Success"
},
{
"fromIndex": 11,
"toIndex": 6,
"type": "True"
},
{
"fromIndex": 12,
"toIndex": 9,
"toIndex": 12,
"type": "Success"
},
{
"fromIndex": 13,
"toIndex": 14,
"type": "Success"
},
{
"fromIndex": 14,
"toIndex": 15,
"type": "Success"
},
{
"fromIndex": 16,
"toIndex": 0,
"type": "False"
},
{
"fromIndex": 16,
"toIndex": 17,
"fromIndex": 13,
"toIndex": 14,
"type": "True"
},
{
"fromIndex": 17,
"toIndex": 13,
"fromIndex": 14,
"toIndex": 10,
"type": "Success"
}
],
"ruleChainConnections": null
}
}
}

11
monitoring/src/main/resources/tb-monitoring.yml

@ -57,6 +57,8 @@ monitoring:
queue: '${MQTT_TRANSPORT_USED_QUEUE:Main}'
# Whether to monitor IPs associated with the domain from base url
check_domain_ips: '${MQTT_TRANSPORT_CHECK_DOMAIN_IPS:false}'
# Prefix for the target device name
name_prefix: '${MQTT_TRANSPORT_TARGET_NAME_PREFIX:}'
# To add more targets, use following environment variables:
# monitoring.transports.mqtt.targets[1].base_url, monitoring.transports.mqtt.targets[2].base_url, etc.
@ -72,6 +74,8 @@ monitoring:
queue: '${COAP_TRANSPORT_USED_QUEUE:Main}'
# Whether to monitor IPs associated with the domain from base url
check_domain_ips: '${COAP_TRANSPORT_CHECK_DOMAIN_IPS:false}'
# Prefix for the target device name
name_prefix: '${COAP_TRANSPORT_TARGET_NAME_PREFIX:}'
# To add more targets, use following environment variables:
# monitoring.transports.coap.targets[1].base_url, monitoring.transports.coap.targets[2].base_url, etc.
@ -87,6 +91,8 @@ monitoring:
queue: '${HTTP_TRANSPORT_USED_QUEUE:Main}'
# Whether to monitor IPs associated with the domain from base url
check_domain_ips: '${HTTP_TRANSPORT_CHECK_DOMAIN_IPS:false}'
# Prefix for the target device name
name_prefix: '${HTTP_TRANSPORT_TARGET_NAME_PREFIX:}'
# To add more targets, use following environment variables:
# monitoring.transports.http.targets[1].base_url, monitoring.transports.http.targets[2].base_url, etc.
@ -102,12 +108,17 @@ monitoring:
queue: '${LWM2M_TRANSPORT_USED_QUEUE:Main}'
# Whether to monitor IPs associated with the domain from base url
check_domain_ips: '${LWM2M_TRANSPORT_CHECK_DOMAIN_IPS:false}'
# Prefix for the target device name
name_prefix: '${LWM2M_TRANSPORT_TARGET_NAME_PREFIX:}'
# To add more targets, use following environment variables:
# monitoring.transports.lwm2m.targets[1].base_url, monitoring.transports.lwm2m.targets[2].base_url, etc.
edqs:
enabled: "${EDQS_MONITORING_ENABLED:false}"
calculated_fields:
enabled: "${CALCULATED_FIELDS_MONITORING_ENABLED:true}"
notifications:
message_prefix: '${NOTIFICATION_MESSAGE_PREFIX:}'
slack:

12
msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java

@ -55,8 +55,8 @@ public class ContainerTestSuite {
private static final String TB_JS_EXECUTOR_LOG_REGEXP = ".*template started.*";
private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(400);
private DockerComposeContainer<?> testContainer;
private ThingsBoardDbInstaller installTb;
private DockerComposeContainer<?> testContainer;
private ThingsBoardDbInstaller installTb;
private boolean isActive;
private static ContainerTestSuite containerTestSuite;
@ -194,7 +194,7 @@ public class ContainerTestSuite {
setActive(true);
} catch (Exception e) {
log.error("Failed to create test container", e);
fail("Failed to create test container");
fail("Failed to create test container", e);
}
}
@ -263,7 +263,7 @@ public class ContainerTestSuite {
log.info("Trying to delete temp dir {}", targetDir);
FileUtils.deleteDirectory(new File(targetDir));
} catch (IOException e) {
log.error("Can't delete temp directory " + targetDir, e);
log.error("Can't delete temp directory {}", targetDir, e);
}
}
@ -286,8 +286,8 @@ public class ContainerTestSuite {
FileUtils.writeStringToFile(file, outputContent, StandardCharsets.UTF_8);
assertThat(FileUtils.readFileToString(file, StandardCharsets.UTF_8), is(outputContent));
} catch (IOException e) {
log.error("failed to update file " + sourceFilename, e);
fail("failed to update file");
log.error("failed to update file {}", sourceFilename, e);
fail("failed to update file", e);
}
}

15
msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java

@ -42,7 +42,6 @@ import org.thingsboard.mqtt.MqttClient;
import org.thingsboard.mqtt.MqttClientCallback;
import org.thingsboard.mqtt.MqttClientConfig;
import org.thingsboard.mqtt.MqttHandler;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.DeviceProfileProvisionType;
@ -82,7 +81,6 @@ import java.util.concurrent.TimeoutException;
import static org.assertj.core.api.Assertions.assertThat;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.fail;
import static org.thingsboard.server.common.data.DataConstants.DEVICE;
import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE;
import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultDevicePrototype;
@ -301,7 +299,7 @@ public class MqttClientTest extends AbstractContainerTest {
assertThat(Objects.requireNonNull(requestFromServer).getMessage()).isEqualTo("{\"method\":\"getValue\",\"params\":true}");
Integer requestId = Integer.valueOf(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length()));
int requestId = Integer.parseInt(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length()));
JsonObject clientResponse = new JsonObject();
clientResponse.addProperty("response", "someResponse");
// Send a response to the server's RPC request
@ -340,7 +338,7 @@ public class MqttClientTest extends AbstractContainerTest {
assertThat(Objects.requireNonNull(requestFromServer).getMessage()).isEqualTo("{\"method\":\"getValue\",\"params\":true}");
Integer requestId = Integer.valueOf(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length()));
int requestId = Integer.parseInt(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length()));
JsonObject clientResponse = new JsonObject();
clientResponse.addProperty("response", "someResponse");
// Send a response to the server's RPC request
@ -520,13 +518,13 @@ public class MqttClientTest extends AbstractContainerTest {
mqttClient.on("/provision/response", listener, MqttQoS.AT_LEAST_ONCE).get(3 * timeoutMultiplier, TimeUnit.SECONDS);
TimeUnit.SECONDS.sleep(2 * timeoutMultiplier);
assertThat(subAckResult[0]).isNotNull();
assertThat(MqttReasonCodes.SubAck.GRANTED_QOS_1.equals(subAckResult[0]));
assertThat(MqttReasonCodes.SubAck.GRANTED_QOS_1).isEqualTo(subAckResult[0]);
subAckResult[0] = null;
mqttClient.on("v1/devices/me/attributes", listener, MqttQoS.AT_LEAST_ONCE).get(3 * timeoutMultiplier, TimeUnit.SECONDS);
TimeUnit.SECONDS.sleep(2 * timeoutMultiplier);
assertThat(subAckResult[0]).isNotNull();
assertThat(MqttReasonCodes.SubAck.TOPIC_FILTER_INVALID.equals(subAckResult[0]));
assertThat(MqttReasonCodes.SubAck.TOPIC_FILTER_INVALID).isEqualTo(subAckResult[0]);
testRestClient.deleteDeviceIfExists(device.getId());
updateDeviceProfileWithProvisioningStrategy(deviceProfile, DeviceProfileProvisionType.DISABLED);
@ -596,7 +594,7 @@ public class MqttClientTest extends AbstractContainerTest {
.await()
.alias("Check device disconnect.")
.atMost(TIMEOUT*timeoutMultiplier, TimeUnit.SECONDS)
.until(() -> returnCodeByteValue.size() > 0);
.until(() -> !returnCodeByteValue.isEmpty());
assertThat(returnCodeByteValueSecondClient).isEmpty();
assertThat(returnCodeByteValue).isNotEmpty();
@ -663,7 +661,7 @@ public class MqttClientTest extends AbstractContainerTest {
.stream()
.filter(RuleChain::isRoot)
.findFirst();
if (!defaultRuleChain.isPresent()) {
if (defaultRuleChain.isEmpty()) {
fail("Root rule chain wasn't found");
}
return defaultRuleChain.get().getId();
@ -717,6 +715,7 @@ public class MqttClientTest extends AbstractContainerTest {
clientConfig.setClientId("MQTT client from test");
clientConfig.setUsername(username);
clientConfig.setProtocolVersion(mqttVersion);
clientConfig.setRetransmissionConfig(new MqttClientConfig.RetransmissionConfig(3, 5000L, 0.15d)); // same as defaults in thingsboard.yml as of time of this writing
MqttClient mqttClient = MqttClient.create(clientConfig, listener, handlerExecutor);
if (connect) {
mqttClient.connect(TRANSPORT_HOST, TRANSPORT_PORT).get();

14
msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java

@ -39,7 +39,6 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.mqtt.MqttClient;
import org.thingsboard.mqtt.MqttClientConfig;
import org.thingsboard.mqtt.MqttHandler;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.id.DeviceId;
@ -65,7 +64,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat;
import static org.thingsboard.server.common.data.DataConstants.DEVICE;
import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE;
import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultGatewayPrototype;
@ -76,7 +74,6 @@ public class MqttGatewayClientTest extends AbstractContainerTest {
private MqttClient mqttClient;
private Device createdDevice;
private MqttMessageListener listener;
private JsonParser jsonParser = new JsonParser();
AbstractListeningExecutor handlerExecutor;
@ -100,7 +97,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest {
}
@AfterMethod
public void removeGateway() {
public void removeGateway() {
testRestClient.deleteDeviceIfExists(this.gatewayDevice.getId());
testRestClient.deleteDeviceIfExists(this.createdDevice.getId());
this.listener = null;
@ -197,7 +194,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest {
mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(requestData.toString().getBytes())).get();
event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS);
JsonObject responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject();
JsonObject responseData = JsonParser.parseString(Objects.requireNonNull(event).getMessage()).getAsJsonObject();
assertThat(responseData.has("value")).isTrue();
assertThat(responseData.get("value").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString());
@ -213,7 +210,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest {
mqttClient.on("v1/gateway/attributes/response", listener, MqttQoS.AT_LEAST_ONCE).get();
mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(requestData.toString().getBytes())).get();
event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS);
responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject();
responseData = JsonParser.parseString(Objects.requireNonNull(event).getMessage()).getAsJsonObject();
assertThat(responseData.has("values")).isTrue();
assertThat(responseData.get("values").getAsJsonObject().get("attr1").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString());
@ -231,7 +228,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest {
mqttClient.on("v1/gateway/attributes/response", listener, MqttQoS.AT_LEAST_ONCE).get();
mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(requestData.toString().getBytes())).get();
event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS);
responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject();
responseData = JsonParser.parseString(Objects.requireNonNull(event).getMessage()).getAsJsonObject();
assertThat(responseData.has("values")).isTrue();
assertThat(responseData.get("values").getAsJsonObject().get("attr1").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString());
@ -390,7 +387,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest {
mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(gatewayAttributesRequest.toString().getBytes())).get();
MqttEvent clientAttributeEvent = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS);
assertThat(clientAttributeEvent).isNotNull();
JsonObject responseMessage = new JsonParser().parse(Objects.requireNonNull(clientAttributeEvent).getMessage()).getAsJsonObject();
JsonObject responseMessage = JsonParser.parseString(Objects.requireNonNull(clientAttributeEvent).getMessage()).getAsJsonObject();
assertThat(responseMessage.get("id").getAsInt()).isEqualTo(messageId);
assertThat(responseMessage.get("device").getAsString()).isEqualTo(createdDevice.getName());
@ -427,6 +424,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest {
clientConfig.setOwnerId(getOwnerId());
clientConfig.setClientId("MQTT client from test");
clientConfig.setUsername(deviceCredentials.getCredentialsId());
clientConfig.setRetransmissionConfig(new MqttClientConfig.RetransmissionConfig(3, 5000L, 0.15d)); // same as defaults in thingsboard.yml as of time of this writing
MqttClient mqttClient = MqttClient.create(clientConfig, listener, handlerExecutor);
mqttClient.connect("localhost", 1883).get();
return mqttClient;

5
msa/vc-executor/src/main/resources/tb-vc-executor.yml

@ -124,6 +124,11 @@ queue:
# tb_rule_engine.sq:
# - key: max.poll.records
# value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}"
# If you override any default Kafka topic name using environment variables, you must also specify the related consumer properties
# for the new topic in `consumer-properties-per-topic-inline`. Otherwise, the topic will not inherit its expected configuration (e.g., max.poll.records, timeouts, etc).
# Format: "topic1:key1=value1,key2=value2;topic2:key=value"
# Example: "tb_core_modified.notifications:max.poll.records=10;tb_edge_modified:max.poll.records=10,enable.auto.commit=true"
consumer-properties-per-topic-inline: "${TB_QUEUE_KAFKA_CONSUMER_PROPERTIES_PER_TOPIC_INLINE:}"
other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000"
other: # DEPRECATED. In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside
# - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms

20
netty-mqtt/pom.xml

@ -87,6 +87,26 @@
<artifactId>awaitility</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>software.xdev</groupId>
<artifactId>testcontainers-junit4-mock</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>hivemq</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>

24
netty-mqtt/src/main/java/org/thingsboard/mqtt/MaxRetransmissionsReachedException.java

@ -0,0 +1,24 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.mqtt;
public class MaxRetransmissionsReachedException extends RuntimeException {
public MaxRetransmissionsReachedException(String message) {
super(message);
}
}

30
netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttChannelHandler.java

@ -57,7 +57,7 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler<MqttMessage>
}
@Override
protected void channelRead0(ChannelHandlerContext ctx, MqttMessage msg) throws Exception {
protected void channelRead0(ChannelHandlerContext ctx, MqttMessage msg) {
if (msg.decoderResult().isSuccess()) {
switch (msg.fixedHeader().messageType()) {
case CONNACK:
@ -120,6 +120,7 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler<MqttMessage>
this.client.getClientConfig().getUsername(),
this.client.getClientConfig().getPassword() != null ? this.client.getClientConfig().getPassword().getBytes(CharsetUtil.UTF_8) : null
);
log.debug("{} Sending CONNECT", client.getClientConfig().getOwnerId());
ctx.channel().writeAndFlush(new MqttConnectMessage(fixedHeader, variableHeader, payload));
}
@ -173,6 +174,7 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler<MqttMessage>
}
private void handleConack(Channel channel, MqttConnAckMessage message) {
log.debug("{} Handling CONNACK", client.getClientConfig().getOwnerId());
switch (message.variableHeader().connectReturnCode()) {
case CONNECTION_ACCEPTED:
this.connectFuture.setSuccess(new MqttConnectResult(true, MqttConnectReturnCode.CONNECTION_ACCEPTED, channel.closeFuture()));
@ -219,9 +221,9 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler<MqttMessage>
}
pendingSubscription.onSubackReceived();
for (MqttPendingSubscription.MqttPendingHandler handler : pendingSubscription.getHandlers()) {
MqttSubscription subscription = new MqttSubscription(pendingSubscription.getTopic(), handler.getHandler(), handler.isOnce());
MqttSubscription subscription = new MqttSubscription(pendingSubscription.getTopic(), handler.handler(), handler.once());
this.client.getSubscriptions().put(pendingSubscription.getTopic(), subscription);
this.client.getHandlerToSubscription().put(handler.getHandler(), subscription);
this.client.getHandlerToSubscription().put(handler.handler(), subscription);
}
this.client.getPendingSubscribeTopics().remove(pendingSubscription.getTopic());
@ -282,17 +284,16 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler<MqttMessage>
}
private void handlePuback(MqttPubAckMessage message) {
MqttPendingPublish pendingPublish = this.client.getPendingPublishes().get(message.variableHeader().messageId());
if (pendingPublish == null) {
return;
}
pendingPublish.getFuture().setSuccess(null);
pendingPublish.onPubackReceived();
this.client.getPendingPublishes().remove(message.variableHeader().messageId());
pendingPublish.getPayload().release();
if (this.client.getCallback() != null) {
this.client.getCallback().onPubAck(message);
}
log.trace("{} Handling PUBACK", client.getClientConfig().getOwnerId());
client.getPendingPublishes().computeIfPresent(message.variableHeader().messageId(), (__, pendingPublish) -> {
pendingPublish.getFuture().setSuccess(null);
pendingPublish.onPubackReceived();
pendingPublish.getPayload().release();
if (client.getCallback() != null) {
client.getCallback().onPubAck(message);
}
return null;
});
}
private void handlePubrec(Channel channel, MqttMessage message) {
@ -335,6 +336,7 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler<MqttMessage>
}
private void handleDisconnect(MqttMessage message) {
log.debug("{} Handling DISCONNECT", client.getClientConfig().getOwnerId());
if (this.client.getCallback() != null) {
this.client.getCallback().onDisconnect(message);
}

2
netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClient.java

@ -184,7 +184,7 @@ public interface MqttClient {
* @param config The config object to use while looking for settings
* @param defaultHandler The handler for incoming messages that do not match any topic subscriptions
*/
static MqttClient create(MqttClientConfig config, MqttHandler defaultHandler, ListeningExecutor handlerExecutor){
static MqttClient create(MqttClientConfig config, MqttHandler defaultHandler, ListeningExecutor handlerExecutor) {
return new MqttClientImpl(config, defaultHandler, handlerExecutor);
}

20
netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientConfig.java

@ -47,6 +47,26 @@ public final class MqttClientConfig {
private long reconnectDelay = 1L;
private int maxBytesInMessage = 8092;
@Getter
@Setter
private RetransmissionConfig retransmissionConfig;
public record RetransmissionConfig(int maxAttempts, long initialDelayMillis, double jitterFactor) {
public RetransmissionConfig {
if (maxAttempts < 0) {
throw new IllegalArgumentException("Max retransmission attempts (maxAttempts) must be zero or greater, but was " + maxAttempts);
}
if (initialDelayMillis < 0) {
throw new IllegalArgumentException("Initial retransmission delay (initialDelayMillis) must be zero or greater, but was " + initialDelayMillis);
}
if (jitterFactor < 0) {
throw new IllegalArgumentException("Jitter factor (jitterFactor) must be zero or greater, but was " + jitterFactor);
}
}
}
public MqttClientConfig() {
this(null);
}

82
netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java

@ -17,6 +17,7 @@ package org.thingsboard.mqtt;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import io.netty.bootstrap.Bootstrap;
import io.netty.buffer.ByteBuf;
import io.netty.channel.Channel;
@ -384,8 +385,33 @@ final class MqttClientImpl implements MqttClient {
MqttFixedHeader fixedHeader = new MqttFixedHeader(MqttMessageType.PUBLISH, false, qos, retain, 0);
MqttPublishVariableHeader variableHeader = new MqttPublishVariableHeader(topic, getNewMessageId().messageId());
MqttPublishMessage message = new MqttPublishMessage(fixedHeader, variableHeader, payload);
MqttPendingPublish pendingPublish = new MqttPendingPublish(variableHeader.packetId(), future,
payload.retain(), message, qos, () -> !pendingPublishes.containsKey(variableHeader.packetId()));
final var pendingPublish = MqttPendingPublish.builder()
.messageId(variableHeader.packetId())
.future(future)
.payload(payload.retain())
.message(message)
.qos(qos)
.ownerId(clientConfig.getOwnerId())
.retransmissionConfig(clientConfig.getRetransmissionConfig())
.pendingOperation(new PendingOperation() {
@Override
public boolean isCancelled() {
return !pendingPublishes.containsKey(variableHeader.packetId());
}
@Override
public void onMaxRetransmissionAttemptsReached() {
pendingPublishes.computeIfPresent(variableHeader.packetId(), (__, pendingPublish) -> {
var message = "Unable to deliver publish message due to max retransmission attempts (%s) being reached for client '%s' on topic '%s' (message ID: %d)"
.formatted(clientConfig.getRetransmissionConfig().maxAttempts(), clientConfig.getClientId(), topic, variableHeader.packetId());
pendingPublish.getFuture().tryFailure(new MaxRetransmissionsReachedException(message));
pendingPublish.getPayload().release();
return null;
});
}
}).build();
this.pendingPublishes.put(pendingPublish.getMessageId(), pendingPublish);
ChannelFuture channelFuture = this.sendAndFlushPacket(message);
@ -499,9 +525,30 @@ final class MqttClientImpl implements MqttClient {
MqttSubscribePayload payload = new MqttSubscribePayload(Collections.singletonList(subscription));
MqttSubscribeMessage message = new MqttSubscribeMessage(fixedHeader, variableHeader, payload);
final MqttPendingSubscription pendingSubscription = new MqttPendingSubscription(future, topic, message,
() -> !pendingSubscriptions.containsKey(variableHeader.messageId()));
pendingSubscription.addHandler(handler, once);
final var pendingSubscription = MqttPendingSubscription.builder()
.future(future)
.topic(topic)
.handlers(Sets.newHashSet(new MqttPendingSubscription.MqttPendingHandler(handler, once)))
.subscribeMessage(message)
.ownerId(clientConfig.getOwnerId())
.retransmissionConfig(clientConfig.getRetransmissionConfig())
.pendingOperation(new PendingOperation() {
@Override
public boolean isCancelled() {
return !pendingSubscriptions.containsKey(variableHeader.messageId());
}
@Override
public void onMaxRetransmissionAttemptsReached() {
pendingSubscriptions.computeIfPresent(variableHeader.messageId(), (__, pendingSubscription) -> {
var message = "Unable to deliver subscribe message due to max retransmission attempts (%s) being reached for client '%s' on topic '%s' (message ID: %d)"
.formatted(clientConfig.getRetransmissionConfig().maxAttempts(), clientConfig.getClientId(), topic, variableHeader.messageId());
pendingSubscription.getFuture().tryFailure(new MaxRetransmissionsReachedException(message));
return null;
});
}
}).build();
this.pendingSubscriptions.put(variableHeader.messageId(), pendingSubscription);
this.pendingSubscribeTopics.add(topic);
pendingSubscription.setSent(this.sendAndFlushPacket(message) != null); //If not sent, we will send it when the connection is opened
@ -518,8 +565,29 @@ final class MqttClientImpl implements MqttClient {
MqttUnsubscribePayload payload = new MqttUnsubscribePayload(Collections.singletonList(topic));
MqttUnsubscribeMessage message = new MqttUnsubscribeMessage(fixedHeader, variableHeader, payload);
MqttPendingUnsubscription pendingUnsubscription = new MqttPendingUnsubscription(promise, topic, message,
() -> !pendingServerUnsubscribes.containsKey(variableHeader.messageId()));
final var pendingUnsubscription = MqttPendingUnsubscription.builder()
.future(promise)
.topic(topic)
.unsubscribeMessage(message)
.ownerId(clientConfig.getOwnerId())
.retransmissionConfig(clientConfig.getRetransmissionConfig())
.pendingOperation(new PendingOperation() {
@Override
public boolean isCancelled() {
return !pendingServerUnsubscribes.containsKey(variableHeader.messageId());
}
@Override
public void onMaxRetransmissionAttemptsReached() {
pendingServerUnsubscribes.computeIfPresent(variableHeader.messageId(), (__, pendingUnsubscription) -> {
var message = "Unable to deliver unsubscribe message due to max retransmission attempts (%s) being reached for client '%s' on topic '%s' (message ID: %d)"
.formatted(clientConfig.getRetransmissionConfig().maxAttempts(), clientConfig.getClientId(), topic, variableHeader.messageId());
pendingUnsubscription.getFuture().tryFailure(new MaxRetransmissionsReachedException(message));
return null;
});
}
}).build();
this.pendingServerUnsubscribes.put(variableHeader.messageId(), pendingUnsubscription);
pendingUnsubscription.startRetransmissionTimer(this.eventLoop.next(), this::sendAndFlushPacket);

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save