Browse Source

Merge with develop/2.5.5

release-2.5
Igor Kulikov 6 years ago
parent
commit
36d75c08f6
  1. 2
      .gitignore
  2. 4
      application/pom.xml
  3. 3
      application/src/main/data/json/demo/rule_chains/root_rule_chain.json
  4. 3
      application/src/main/data/json/tenant/rule_chains/root_rule_chain.json
  5. 1
      application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql
  6. 48
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
  7. 2
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java
  8. 2
      application/src/main/java/org/thingsboard/server/controller/AlarmController.java
  9. 7
      application/src/main/java/org/thingsboard/server/controller/AuthController.java
  10. 11
      application/src/main/java/org/thingsboard/server/controller/TelemetryController.java
  11. 10
      application/src/main/java/org/thingsboard/server/controller/UserController.java
  12. 5
      application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
  13. 1
      application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java
  14. 1
      application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java
  15. 6
      application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java
  16. 2
      application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java
  17. 4
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
  18. 12
      application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java
  19. 60
      application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java
  20. 85
      application/src/main/java/org/thingsboard/server/service/queue/TbMsgProfilerInfo.java
  21. 75
      application/src/main/java/org/thingsboard/server/service/queue/TbRuleNodeProfilerInfo.java
  22. 4
      application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java
  23. 8
      application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java
  24. 2
      application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationFailureHandler.java
  25. 1
      application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java
  26. 24
      application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java
  27. 5
      application/src/main/java/org/thingsboard/server/service/security/system/SystemSecurityService.java
  28. 3
      application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java
  29. 4
      application/src/main/java/org/thingsboard/server/service/state/DeviceStateData.java
  30. 7
      application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java
  31. 2
      application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java
  32. 11
      application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java
  33. 19
      application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java
  34. 2
      application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java
  35. 3
      application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java
  36. 3
      application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java
  37. 2
      application/src/main/resources/logback.xml
  38. 19
      application/src/main/resources/thingsboard.yml
  39. 1
      application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java
  40. 2
      application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java
  41. 4
      common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeInfo.java
  42. 8
      common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java
  43. 10
      common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java
  44. 15
      common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java
  45. 2
      common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java
  46. 9
      common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java
  47. 1
      common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java
  48. 5
      common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java
  49. 4
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java
  50. 17
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java
  51. 5
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java
  52. 2
      dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java
  53. 17
      dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java
  54. 33
      dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java
  55. 1
      dao/src/main/resources/sql/schema-ts-psql.sql
  56. 90
      dao/src/test/java/org/thingsboard/server/dao/service/BaseRuleChainServiceTest.java
  57. 2
      dao/src/test/resources/sql-test.properties
  58. 2
      msa/js-executor/config/default.yml
  59. 2599
      msa/js-executor/package-lock.json
  60. 4
      msa/js-executor/package.json
  61. 6
      msa/js-executor/queue/awsSqsTemplate.js
  62. 33
      msa/js-executor/queue/kafkaTemplate.js
  63. 2
      msa/web-ui/package.json
  64. 33
      pom.xml
  65. 2
      rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java
  66. 2
      rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java
  67. 5
      rule-engine/rule-engine-components/pom.xml
  68. 15
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java
  69. 17
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java
  70. 3
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeConfiguration.java
  71. 6
      rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js
  72. 7
      rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java
  73. 1
      tools/pom.xml
  74. 10
      transport/coap/src/main/resources/tb-coap-transport.yml
  75. 10
      transport/http/src/main/resources/tb-http-transport.yml
  76. 12
      transport/mqtt/src/main/resources/tb-mqtt-transport.yml
  77. 2
      ui/package.json
  78. 8
      ui/src/app/admin/general-settings.tpl.html
  79. 2
      ui/src/app/locale/locale.constant-en_US.json
  80. 17
      ui/src/app/widget/lib/rpc/knob.directive.js

2
.gitignore

@ -33,3 +33,5 @@ pom.xml.versionsBackup
**/.env
.instance_id
rebuild-docker.sh
.run/**
.run

4
application/pom.xml

@ -145,10 +145,6 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-websocket</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-oauth2</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-oauth2-client</artifactId>

3
application/src/main/data/json/demo/rule_chains/root_rule_chain.json

@ -43,7 +43,8 @@
"name": "Save Client Attributes",
"debugMode": false,
"configuration": {
"scope": "CLIENT_SCOPE"
"scope": "CLIENT_SCOPE",
"notifyDevice": "false"
}
},
{

3
application/src/main/data/json/tenant/rule_chains/root_rule_chain.json

@ -31,7 +31,8 @@
"name": "Save Client Attributes",
"debugMode": false,
"configuration": {
"scope": "CLIENT_SCOPE"
"scope": "CLIENT_SCOPE",
"notifyDevice": "false"
}
},
{

1
application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql

@ -64,6 +64,7 @@ BEGIN
AND tablename like 'ts_kv_' || '%'
AND tablename != 'ts_kv_latest'
AND tablename != 'ts_kv_dictionary'
AND tablename != 'ts_kv_indefinite'
LOOP
IF partition != partition_by_max_ttl_date THEN
IF partition_year IS NOT NULL THEN

48
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java

@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import io.netty.channel.EventLoopGroup;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.util.StringUtils;
import org.thingsboard.common.util.ListeningExecutor;
import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.rule.engine.api.RuleEngineRpcService;
@ -43,6 +44,7 @@ import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.queue.ServiceQueue;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.dao.alarm.AlarmService;
@ -66,7 +68,6 @@ import org.thingsboard.server.service.script.RuleNodeJsScriptEngine;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
/**
@ -104,6 +105,7 @@ class DefaultTbContext implements TbContext {
if (nodeCtx.getSelf().isDebugMode()) {
relationTypes.forEach(relationType -> mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), msg, relationType, th));
}
msg.getCallback().onProcessingEnd(nodeCtx.getSelf().getId());
nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), relationTypes, msg, th != null ? th.getMessage() : null));
}
@ -121,7 +123,7 @@ class DefaultTbContext implements TbContext {
@Override
public void enqueue(TbMsg tbMsg, String queueName, Runnable onSuccess, Consumer<Throwable> onFailure) {
TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator());
TopicPartitionInfo tpi = resolvePartition(tbMsg, queueName);
enqueue(tpi, tbMsg, onFailure, onSuccess);
}
@ -138,46 +140,57 @@ class DefaultTbContext implements TbContext {
@Override
public void enqueueForTellFailure(TbMsg tbMsg, String failureMessage) {
TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator());
TopicPartitionInfo tpi = resolvePartition(tbMsg);
enqueueForTellNext(tpi, tbMsg, Collections.singleton(TbRelationTypes.FAILURE), failureMessage, null, null);
}
@Override
public void enqueueForTellNext(TbMsg tbMsg, String relationType) {
TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator());
TopicPartitionInfo tpi = resolvePartition(tbMsg);
enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, null, null);
}
@Override
public void enqueueForTellNext(TbMsg tbMsg, Set<String> relationTypes) {
TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator());
TopicPartitionInfo tpi = resolvePartition(tbMsg);
enqueueForTellNext(tpi, tbMsg, relationTypes, null, null, null);
}
@Override
public void enqueueForTellNext(TbMsg tbMsg, String relationType, Runnable onSuccess, Consumer<Throwable> onFailure) {
TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator());
TopicPartitionInfo tpi = resolvePartition(tbMsg);
enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, onSuccess, onFailure);
}
@Override
public void enqueueForTellNext(TbMsg tbMsg, Set<String> relationTypes, Runnable onSuccess, Consumer<Throwable> onFailure) {
TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator());
TopicPartitionInfo tpi = resolvePartition(tbMsg);
enqueueForTellNext(tpi, tbMsg, relationTypes, null, onSuccess, onFailure);
}
@Override
public void enqueueForTellNext(TbMsg tbMsg, String queueName, String relationType, Runnable onSuccess, Consumer<Throwable> onFailure) {
TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator());
TopicPartitionInfo tpi = resolvePartition(tbMsg, queueName);
enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, onSuccess, onFailure);
}
@Override
public void enqueueForTellNext(TbMsg tbMsg, String queueName, Set<String> relationTypes, Runnable onSuccess, Consumer<Throwable> onFailure) {
TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator());
TopicPartitionInfo tpi = resolvePartition(tbMsg, queueName);
enqueueForTellNext(tpi, tbMsg, relationTypes, null, onSuccess, onFailure);
}
private TopicPartitionInfo resolvePartition(TbMsg tbMsg, String queueName) {
if (StringUtils.isEmpty(queueName)) {
queueName = ServiceQueue.MAIN;
}
return mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator());
}
private TopicPartitionInfo resolvePartition(TbMsg tbMsg) {
return resolvePartition(tbMsg, tbMsg.getQueueName());
}
private void enqueueForTellNext(TopicPartitionInfo tpi, TbMsg source, Set<String> relationTypes, String failureMessage, Runnable onSuccess, Consumer<Throwable> onFailure) {
RuleChainId ruleChainId = nodeCtx.getSelf().getRuleChainId();
RuleNodeId ruleNodeId = nodeCtx.getSelf().getId();
@ -202,6 +215,7 @@ class DefaultTbContext implements TbContext {
if (nodeCtx.getSelf().isDebugMode()) {
mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), tbMsg, "ACK", null);
}
tbMsg.getCallback().onProcessingEnd(nodeCtx.getSelf().getId());
tbMsg.getCallback().onSuccess();
}
@ -238,26 +252,26 @@ class DefaultTbContext implements TbContext {
}
public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) {
return entityCreatedMsg(customer, customer.getId(), ruleNodeId);
return entityActionMsg(customer, customer.getId(), ruleNodeId, DataConstants.ENTITY_CREATED);
}
public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) {
return entityCreatedMsg(device, device.getId(), ruleNodeId);
return entityActionMsg(device, device.getId(), ruleNodeId, DataConstants.ENTITY_CREATED);
}
public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) {
return entityCreatedMsg(asset, asset.getId(), ruleNodeId);
return entityActionMsg(asset, asset.getId(), ruleNodeId, DataConstants.ENTITY_CREATED);
}
public TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId) {
return entityCreatedMsg(alarm, alarm.getId(), ruleNodeId);
public TbMsg alarmActionMsg(Alarm alarm, RuleNodeId ruleNodeId, String action) {
return entityActionMsg(alarm, alarm.getId(), ruleNodeId, action);
}
public <E, I extends EntityId> TbMsg entityCreatedMsg(E entity, I id, RuleNodeId ruleNodeId) {
public <E, I extends EntityId> TbMsg entityActionMsg(E entity, I id, RuleNodeId ruleNodeId, String action) {
try {
return TbMsg.newMsg(DataConstants.ENTITY_CREATED, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity)));
return TbMsg.newMsg(action, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity)));
} catch (JsonProcessingException | IllegalArgumentException e) {
throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " created msg: " + e);
throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " " + action + " msg: " + e);
}
}

2
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java

@ -103,7 +103,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
}
void onRuleChainToRuleNodeMsg(RuleChainToRuleNodeMsg msg) throws Exception {
msg.getMsg().getCallback().visit(info);
msg.getMsg().getCallback().onProcessingStart(info);
checkActive(msg.getMsg());
if (ruleNode.isDebugMode()) {
systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), msg.getFromRelationType());

2
application/src/main/java/org/thingsboard/server/controller/AlarmController.java

@ -124,6 +124,7 @@ public class AlarmController extends BaseController {
long ackTs = System.currentTimeMillis();
alarmService.ackAlarm(getCurrentUser().getTenantId(), alarmId, ackTs).get();
alarm.setAckTs(ackTs);
alarm.setStatus(alarm.getStatus().isCleared() ? AlarmStatus.CLEARED_ACK : AlarmStatus.ACTIVE_ACK);
logEntityAction(alarmId, alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_ACK, null);
} catch (Exception e) {
throw handleException(e);
@ -141,6 +142,7 @@ public class AlarmController extends BaseController {
long clearTs = System.currentTimeMillis();
alarmService.clearAlarm(getCurrentUser().getTenantId(), alarmId, null, clearTs).get();
alarm.setClearTs(clearTs);
alarm.setStatus(alarm.getStatus().isAck() ? AlarmStatus.CLEARED_ACK : AlarmStatus.CLEARED_UNACK);
logEntityAction(alarmId, alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_CLEAR, null);
} catch (Exception e) {
throw handleException(e);

7
application/src/main/java/org/thingsboard/server/controller/AuthController.java

@ -171,7 +171,8 @@ public class AuthController extends BaseController {
try {
String email = resetPasswordByEmailRequest.get("email").asText();
UserCredentials userCredentials = userService.requestPasswordReset(TenantId.SYS_TENANT_ID, email);
String baseUrl = MiscUtils.constructBaseUrl(request);
User user = userService.findUserById(TenantId.SYS_TENANT_ID, userCredentials.getUserId());
String baseUrl = systemSecurityService.getBaseUrl(user.getTenantId(), user.getCustomerId(), request);
String resetUrl = String.format("%s/api/noauth/resetPassword?resetToken=%s", baseUrl,
userCredentials.getResetToken());
@ -219,7 +220,7 @@ public class AuthController extends BaseController {
User user = userService.findUserById(TenantId.SYS_TENANT_ID, credentials.getUserId());
UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail());
SecurityUser securityUser = new SecurityUser(user, credentials.isEnabled(), principal);
String baseUrl = MiscUtils.constructBaseUrl(request);
String baseUrl = systemSecurityService.getBaseUrl(user.getTenantId(), user.getCustomerId(), request);
String loginUrl = String.format("%s/login", baseUrl);
String email = user.getEmail();
@ -266,7 +267,7 @@ public class AuthController extends BaseController {
User user = userService.findUserById(TenantId.SYS_TENANT_ID, userCredentials.getUserId());
UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail());
SecurityUser securityUser = new SecurityUser(user, userCredentials.isEnabled(), principal);
String baseUrl = MiscUtils.constructBaseUrl(request);
String baseUrl = systemSecurityService.getBaseUrl(user.getTenantId(), user.getCustomerId(), request);
String loginUrl = String.format("%s/login", baseUrl);
String email = user.getEmail();
mailService.sendPasswordWasResetEmail(loginUrl, email);

11
application/src/main/java/org/thingsboard/server/controller/TelemetryController.java

@ -197,19 +197,21 @@ public class TelemetryController extends BaseController {
@RequestMapping(value = "/{entityType}/{entityId}/values/timeseries", method = RequestMethod.GET, params = {"keys", "startTs", "endTs"})
@ResponseBody
public DeferredResult<ResponseEntity> getTimeseries(
@PathVariable("entityType") String entityType, @PathVariable("entityId") String entityIdStr,
@PathVariable("entityType") String entityType,
@PathVariable("entityId") String entityIdStr,
@RequestParam(name = "keys") String keys,
@RequestParam(name = "startTs") Long startTs,
@RequestParam(name = "endTs") Long endTs,
@RequestParam(name = "interval", defaultValue = "0") Long interval,
@RequestParam(name = "limit", defaultValue = "100") Integer limit,
@RequestParam(name = "agg", defaultValue = "NONE") String aggStr,
@RequestParam(name= "orderBy", defaultValue = "DESC") String orderBy,
@RequestParam(name = "useStrictDataTypes", required = false, defaultValue = "false") Boolean useStrictDataTypes) throws ThingsboardException {
return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.READ_TELEMETRY, entityType, entityIdStr,
(result, tenantId, entityId) -> {
// If interval is 0, convert this to a NONE aggregation, which is probably what the user really wanted
Aggregation agg = interval == 0L ? Aggregation.valueOf(Aggregation.NONE.name()) : Aggregation.valueOf(aggStr);
List<ReadTsKvQuery> queries = toKeysList(keys).stream().map(key -> new BaseReadTsKvQuery(key, startTs, endTs, interval, limit, agg))
List<ReadTsKvQuery> queries = toKeysList(keys).stream().map(key -> new BaseReadTsKvQuery(key, startTs, endTs, interval, limit, agg, orderBy))
.collect(Collectors.toList());
Futures.addCallback(tsService.findAll(tenantId, entityId, queries), getTsKvListCallback(result, useStrictDataTypes), MoreExecutors.directExecutor());
@ -391,6 +393,11 @@ public class TelemetryController extends BaseController {
if (attributes.isEmpty()) {
return getImmediateDeferredResult("No attributes data found in request body!", HttpStatus.BAD_REQUEST);
}
for (AttributeKvEntry attributeKvEntry: attributes) {
if (attributeKvEntry.getKey().isEmpty() || attributeKvEntry.getKey().trim().length() == 0) {
return getImmediateDeferredResult("Key cannot be empty or contains only spaces", HttpStatus.BAD_REQUEST);
}
}
SecurityUser user = getCurrentUser();
return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdSrc, (result, tenantId, entityId) -> {
tsSubService.saveAndNotify(tenantId, entityId, scope, attributes, new FutureCallback<Void>() {

10
application/src/main/java/org/thingsboard/server/controller/UserController.java

@ -52,6 +52,7 @@ import org.thingsboard.server.service.security.model.token.JwtToken;
import org.thingsboard.server.service.security.model.token.JwtTokenFactory;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;
import org.thingsboard.server.service.security.system.SystemSecurityService;
import org.thingsboard.server.utils.MiscUtils;
import javax.servlet.http.HttpServletRequest;
@ -78,6 +79,9 @@ public class UserController extends BaseController {
@Autowired
private RefreshTokenRepository refreshTokenRepository;
@Autowired
private SystemSecurityService systemSecurityService;
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
@RequestMapping(value = "/user/{userId}", method = RequestMethod.GET)
@ -145,7 +149,7 @@ public class UserController extends BaseController {
if (sendEmail) {
SecurityUser authUser = getCurrentUser();
UserCredentials userCredentials = userService.findUserCredentialsByUserId(authUser.getTenantId(), savedUser.getId());
String baseUrl = MiscUtils.constructBaseUrl(request);
String baseUrl = systemSecurityService.getBaseUrl(getTenantId(), getCurrentUser().getCustomerId(), request);
String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl,
userCredentials.getActivateToken());
String email = savedUser.getEmail();
@ -185,7 +189,7 @@ public class UserController extends BaseController {
UserCredentials userCredentials = userService.findUserCredentialsByUserId(getCurrentUser().getTenantId(), user.getId());
if (!userCredentials.isEnabled()) {
String baseUrl = MiscUtils.constructBaseUrl(request);
String baseUrl = systemSecurityService.getBaseUrl(getTenantId(), getCurrentUser().getCustomerId(), request);
String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl,
userCredentials.getActivateToken());
mailService.sendActivationEmail(activateUrl, email);
@ -210,7 +214,7 @@ public class UserController extends BaseController {
SecurityUser authUser = getCurrentUser();
UserCredentials userCredentials = userService.findUserCredentialsByUserId(authUser.getTenantId(), user.getId());
if (!userCredentials.isEnabled()) {
String baseUrl = MiscUtils.constructBaseUrl(request);
String baseUrl = systemSecurityService.getBaseUrl(getTenantId(), getCurrentUser().getCustomerId(), request);
String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl,
userCredentials.getActivateToken());
return activateUrl;

5
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@ -146,6 +146,11 @@ public class ThingsboardInstallService {
databaseTsUpgradeService.upgradeDatabase("2.5.0");
}
case "2.5.4":
log.info("Upgrading ThingsBoard from version 2.5.4 to 2.5.5 ...");
if (databaseTsUpgradeService != null) {
databaseTsUpgradeService.upgradeDatabase("2.5.4");
}
log.info("Updating system data...");

1
application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java

@ -49,6 +49,7 @@ public class CassandraTsDatabaseUpgradeService extends AbstractCassandraDatabase
log.info("Schema updated.");
break;
case "2.5.0":
case "2.5.4":
break;
default:
throw new RuntimeException("Unable to upgrade Cassandra database, unsupported fromVersion: " + fromVersion);

1
application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java

@ -116,6 +116,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
generalSettings.setKey("general");
ObjectNode node = objectMapper.createObjectNode();
node.put("baseUrl", "http://localhost:8080");
node.put("prohibitDifferentUrl", true);
generalSettings.setJsonValue(node);
adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, generalSettings);

6
application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java

@ -195,6 +195,12 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005001");
}
break;
case "2.5.4":
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
log.info("Load Drop Partitions functions ...");
loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL);
}
break;
default:
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
}

2
application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java

@ -177,6 +177,8 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005001");
}
break;
case "2.5.4":
break;
default:
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
}

4
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@ -165,7 +165,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
submitStrategy.init(msgs);
while (!stopped) {
TbMsgPackProcessingContext ctx = new TbMsgPackProcessingContext(submitStrategy);
TbMsgPackProcessingContext ctx = new TbMsgPackProcessingContext(configuration.getName(), submitStrategy);
submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> {
log.trace("[{}] Creating callback for message: {}", id, msg.getValue());
ToRuleEngineMsg toRuleEngineMsg = msg.getValue();
@ -194,6 +194,8 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
if (!ctx.getFailedMap().isEmpty()) {
printFirstOrAll(configuration, ctx, ctx.getFailedMap(), "Failed");
}
ctx.printProfilerStats();
TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result);
if (statsEnabled) {
stats.log(result, decision.isCommit());

12
application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java

@ -49,8 +49,14 @@ public class TbMsgPackCallback implements TbMsgCallback {
}
@Override
public void visit(RuleNodeInfo ruleNodeInfo) {
log.trace("[{}] ON PROCESS: {}", id, ruleNodeInfo);
ctx.visit(id, ruleNodeInfo);
public void onProcessingStart(RuleNodeInfo ruleNodeInfo) {
log.trace("[{}] ON PROCESSING START: {}", id, ruleNodeInfo);
ctx.onProcessingStart(id, ruleNodeInfo);
}
@Override
public void onProcessingEnd(RuleNodeId ruleNodeId) {
log.trace("[{}] ON PROCESSING END: {}", id, ruleNodeId);
ctx.onProcessingEnd(id, ruleNodeId);
}
}

60
application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java

@ -16,6 +16,7 @@
package org.thingsboard.server.service.queue;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.RuleEngineException;
@ -24,6 +25,8 @@ import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy;
import java.util.Comparator;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@ -31,9 +34,13 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@Slf4j
public class TbMsgPackProcessingContext {
private final String queueName;
private final TbRuleEngineSubmitStrategy submitStrategy;
@Getter
private final boolean profilerEnabled;
private final AtomicInteger pendingCount;
private final CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
@Getter
@ -47,14 +54,20 @@ public class TbMsgPackProcessingContext {
private final ConcurrentMap<UUID, RuleNodeInfo> lastRuleNodeMap = new ConcurrentHashMap<>();
public TbMsgPackProcessingContext(TbRuleEngineSubmitStrategy submitStrategy) {
public TbMsgPackProcessingContext(String queueName, TbRuleEngineSubmitStrategy submitStrategy) {
this.queueName = queueName;
this.submitStrategy = submitStrategy;
this.profilerEnabled = log.isDebugEnabled();
this.pendingMap = submitStrategy.getPendingMap();
this.pendingCount = new AtomicInteger(pendingMap.size());
}
public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException {
return processingTimeoutLatch.await(packProcessingTimeout, milliseconds);
boolean success = processingTimeoutLatch.await(packProcessingTimeout, milliseconds);
if (!success && profilerEnabled) {
msgProfilerMap.values().forEach(this::onTimeout);
}
return success;
}
public void onSuccess(UUID id) {
@ -85,12 +98,53 @@ public class TbMsgPackProcessingContext {
}
}
public void visit(UUID id, RuleNodeInfo ruleNodeInfo) {
private final ConcurrentHashMap<UUID, TbMsgProfilerInfo> msgProfilerMap = new ConcurrentHashMap<>();
private final ConcurrentHashMap<UUID, TbRuleNodeProfilerInfo> ruleNodeProfilerMap = new ConcurrentHashMap<>();
public void onProcessingStart(UUID id, RuleNodeInfo ruleNodeInfo) {
lastRuleNodeMap.put(id, ruleNodeInfo);
if (profilerEnabled) {
msgProfilerMap.computeIfAbsent(id, TbMsgProfilerInfo::new).onStart(ruleNodeInfo.getRuleNodeId());
ruleNodeProfilerMap.putIfAbsent(ruleNodeInfo.getRuleNodeId().getId(), new TbRuleNodeProfilerInfo(ruleNodeInfo));
}
}
public void onProcessingEnd(UUID id, RuleNodeId ruleNodeId) {
if (profilerEnabled) {
long processingTime = msgProfilerMap.computeIfAbsent(id, TbMsgProfilerInfo::new).onEnd(ruleNodeId);
if (processingTime > 0) {
ruleNodeProfilerMap.computeIfAbsent(ruleNodeId.getId(), TbRuleNodeProfilerInfo::new).record(processingTime);
}
}
}
public void onTimeout(TbMsgProfilerInfo profilerInfo) {
Map.Entry<UUID, Long> ruleNodeInfo = profilerInfo.onTimeout();
if (ruleNodeInfo != null) {
ruleNodeProfilerMap.computeIfAbsent(ruleNodeInfo.getKey(), TbRuleNodeProfilerInfo::new).record(ruleNodeInfo.getValue());
}
}
public RuleNodeInfo getLastVisitedRuleNode(UUID id) {
return lastRuleNodeMap.get(id);
}
public void printProfilerStats() {
if (profilerEnabled) {
log.debug("Top Rule Nodes by max execution time:");
ruleNodeProfilerMap.values().stream()
.sorted(Comparator.comparingLong(TbRuleNodeProfilerInfo::getMaxExecutionTime).reversed()).limit(5)
.forEach(info -> log.debug("[{}][{}] max execution time: {}. {}", queueName, info.getRuleNodeId(), info.getMaxExecutionTime(), info.getLabel()));
log.info("Top Rule Nodes by avg execution time:");
ruleNodeProfilerMap.values().stream()
.sorted(Comparator.comparingDouble(TbRuleNodeProfilerInfo::getAvgExecutionTime).reversed()).limit(5)
.forEach(info -> log.info("[{}][{}] avg execution time: {}. {}", queueName, info.getRuleNodeId(), info.getAvgExecutionTime(), info.getLabel()));
log.info("Top Rule Nodes by execution count:");
ruleNodeProfilerMap.values().stream()
.sorted(Comparator.comparingInt(TbRuleNodeProfilerInfo::getExecutionCount).reversed()).limit(5)
.forEach(info -> log.info("[{}][{}] execution count: {}. {}", queueName, info.getRuleNodeId(), info.getExecutionCount(), info.getLabel()));
}
}
}

85
application/src/main/java/org/thingsboard/server/service/queue/TbMsgProfilerInfo.java

@ -0,0 +1,85 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.queue;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
import java.util.AbstractMap;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@Slf4j
public class TbMsgProfilerInfo {
private final UUID msgId;
private AtomicLong totalProcessingTime = new AtomicLong();
private Lock stateLock = new ReentrantLock();
private RuleNodeId currentRuleNodeId;
private long stateChangeTime;
public TbMsgProfilerInfo(UUID msgId) {
this.msgId = msgId;
}
public void onStart(RuleNodeId ruleNodeId) {
long currentTime = System.currentTimeMillis();
stateLock.lock();
try {
currentRuleNodeId = ruleNodeId;
stateChangeTime = currentTime;
} finally {
stateLock.unlock();
}
}
public long onEnd(RuleNodeId ruleNodeId) {
long currentTime = System.currentTimeMillis();
stateLock.lock();
try {
if (ruleNodeId.equals(currentRuleNodeId)) {
long processingTime = currentTime - stateChangeTime;
stateChangeTime = currentTime;
totalProcessingTime.addAndGet(processingTime);
currentRuleNodeId = null;
return processingTime;
} else {
log.trace("[{}] Invalid sequence of rule node processing detected. Expected [{}] but was [{}]", msgId, currentRuleNodeId, ruleNodeId);
return 0;
}
} finally {
stateLock.unlock();
}
}
public Map.Entry<UUID, Long> onTimeout() {
long currentTime = System.currentTimeMillis();
stateLock.lock();
try {
if (currentRuleNodeId != null && stateChangeTime > 0) {
long timeoutTime = currentTime - stateChangeTime;
totalProcessingTime.addAndGet(timeoutTime);
return new AbstractMap.SimpleEntry<>(currentRuleNodeId.getId(), timeoutTime);
}
} finally {
stateLock.unlock();
}
return null;
}
}

75
application/src/main/java/org/thingsboard/server/service/queue/TbRuleNodeProfilerInfo.java

@ -0,0 +1,75 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.queue;
import lombok.Getter;
import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
public class TbRuleNodeProfilerInfo {
@Getter
private final UUID ruleNodeId;
@Getter
private final String label;
private AtomicInteger executionCount = new AtomicInteger(0);
private AtomicLong executionTime = new AtomicLong(0);
private AtomicLong maxExecutionTime = new AtomicLong(0);
public TbRuleNodeProfilerInfo(RuleNodeInfo ruleNodeInfo) {
this.ruleNodeId = ruleNodeInfo.getRuleNodeId().getId();
this.label = ruleNodeInfo.toString();
}
public TbRuleNodeProfilerInfo(UUID ruleNodeId) {
this.ruleNodeId = ruleNodeId;
this.label = "";
}
public void record(long processingTime) {
executionCount.incrementAndGet();
executionTime.addAndGet(processingTime);
while (true) {
long value = maxExecutionTime.get();
if (value >= processingTime) {
break;
}
if (maxExecutionTime.compareAndSet(value, processingTime)) {
break;
}
}
}
int getExecutionCount() {
return executionCount.get();
}
long getMaxExecutionTime() {
return maxExecutionTime.get();
}
double getAvgExecutionTime() {
double executionCnt = (double) executionCount.get();
if (executionCnt > 0) {
return executionTime.get() / executionCnt;
} else {
return 0.0;
}
}
}

4
application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java

@ -68,18 +68,20 @@ public class BatchTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitS
int listSize = orderedMsgList.size();
int startIdx = Math.min(packIdx.get() * batchSize, listSize);
int endIdx = Math.min(startIdx + batchSize, listSize);
Map<UUID, TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> tmpPack;
synchronized (pendingPack) {
pendingPack.clear();
for (int i = startIdx; i < endIdx; i++) {
IdMsgPair pair = orderedMsgList.get(i);
pendingPack.put(pair.uuid, pair.msg);
}
tmpPack = new LinkedHashMap<>(pendingPack);
}
int submitSize = pendingPack.size();
if (log.isDebugEnabled() && submitSize > 0) {
log.debug("[{}] submitting [{}] messages to rule engine", queueName, submitSize);
}
pendingPack.forEach(msgConsumer);
tmpPack.forEach(msgConsumer);
}
}

8
application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java

@ -56,7 +56,9 @@ public class TbRuleEngineProcessingStrategyFactory {
private final boolean retryTimeout;
private final int maxRetries;
private final double maxAllowedFailurePercentage;
private final long pauseBetweenRetries;
private final long maxPauseBetweenRetries;
private long pauseBetweenRetries;
private int initialTotalCount;
private int retryCount;
@ -69,6 +71,7 @@ public class TbRuleEngineProcessingStrategyFactory {
this.maxRetries = configuration.getRetries();
this.maxAllowedFailurePercentage = configuration.getFailurePercentage();
this.pauseBetweenRetries = configuration.getPauseBetweenRetries();
this.maxPauseBetweenRetries = configuration.getMaxPauseBetweenRetries();
}
@Override
@ -108,6 +111,9 @@ public class TbRuleEngineProcessingStrategyFactory {
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
if (maxPauseBetweenRetries > pauseBetweenRetries) {
pauseBetweenRetries = Math.min(maxPauseBetweenRetries, pauseBetweenRetries * 2);
}
}
return new TbRuleEngineProcessingDecision(false, toReprocess);
}

2
application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationFailureHandler.java

@ -30,7 +30,7 @@ import java.nio.charset.StandardCharsets;
@Component(value = "oauth2AuthenticationFailureHandler")
@ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true")
public class Oauth2AuthenticationFailureHandler extends SimpleUrlAuthenticationFailureHandler {
public class Oauth2AuthenticationFailureHandler extends SimpleUrlAuthenticationFailureHandler {
@Override
public void onAuthenticationFailure(HttpServletRequest request,

1
application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java

@ -59,7 +59,6 @@ public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationS
public void onAuthenticationSuccess(HttpServletRequest request,
HttpServletResponse response,
Authentication authentication) throws IOException {
String baseUrl = MiscUtils.constructBaseUrl(request);
try {
OAuth2AuthenticationToken token = (OAuth2AuthenticationToken) authentication;

24
application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java

@ -40,17 +40,20 @@ import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.security.model.SecuritySettings;
import org.thingsboard.server.common.data.security.model.UserPasswordPolicy;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.dao.user.UserServiceImpl;
import org.thingsboard.server.service.security.exception.UserPasswordExpiredException;
import org.thingsboard.server.common.data.security.model.SecuritySettings;
import org.thingsboard.server.common.data.security.model.UserPasswordPolicy;
import org.thingsboard.server.utils.MiscUtils;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -146,7 +149,7 @@ public class DefaultSystemSecurityService implements SystemSecurityService {
if (isPositiveInteger(securitySettings.getPasswordPolicy().getPasswordExpirationPeriodDays())) {
if ((userCredentials.getCreatedTime()
+ TimeUnit.DAYS.toMillis(securitySettings.getPasswordPolicy().getPasswordExpirationPeriodDays()))
< System.currentTimeMillis()) {
< System.currentTimeMillis()) {
userCredentials = userService.requestExpiredPasswordReset(tenantId, userCredentials.getId());
throw new UserPasswordExpiredException("User password expired!", userCredentials.getResetToken());
}
@ -197,6 +200,21 @@ public class DefaultSystemSecurityService implements SystemSecurityService {
}
}
@Override
public String getBaseUrl(TenantId tenantId, CustomerId customerId, HttpServletRequest httpServletRequest) {
String baseUrl;
AdminSettings generalSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "general");
JsonNode prohibitDifferentUrl = generalSettings.getJsonValue().get("prohibitDifferentUrl");
if (prohibitDifferentUrl != null && prohibitDifferentUrl.asBoolean()) {
baseUrl = generalSettings.getJsonValue().get("baseUrl").asText();
} else {
baseUrl = MiscUtils.constructBaseUrl(httpServletRequest);
}
return baseUrl;
}
private static boolean isPositiveInteger(Integer val) {
return val != null && val.intValue() > 0;
}

5
application/src/main/java/org/thingsboard/server/service/security/system/SystemSecurityService.java

@ -16,11 +16,14 @@
package org.thingsboard.server.service.security.system;
import org.springframework.security.core.AuthenticationException;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.common.data.security.model.SecuritySettings;
import javax.servlet.http.HttpServletRequest;
public interface SystemSecurityService {
SecuritySettings getSecuritySettings(TenantId tenantId);
@ -31,4 +34,6 @@ public interface SystemSecurityService {
void validatePassword(TenantId tenantId, String password, UserCredentials userCredentials) throws DataValidationException;
String getBaseUrl(TenantId tenantId, CustomerId customerId, HttpServletRequest httpServletRequest);
}

3
application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java

@ -392,7 +392,7 @@ public class DefaultDeviceStateService implements DeviceStateService {
if (stateData != null) {
DeviceState state = stateData.getState();
state.setActive(ts < state.getLastActivityTime() + state.getInactivityTimeout());
if (!state.isActive() && (state.getLastInactivityAlarmTime() == 0L || state.getLastInactivityAlarmTime() < state.getLastActivityTime())) {
if (!state.isActive() && (state.getLastInactivityAlarmTime() == 0L || state.getLastInactivityAlarmTime() < state.getLastActivityTime()) && stateData.getDeviceCreationTime() + state.getInactivityTimeout() < ts) {
state.setLastInactivityAlarmTime(ts);
pushRuleEngineMessage(stateData, INACTIVITY_EVENT);
save(deviceId, INACTIVITY_ALARM_TIME, ts);
@ -479,6 +479,7 @@ public class DefaultDeviceStateService implements DeviceStateService {
return DeviceStateData.builder()
.tenantId(device.getTenantId())
.deviceId(device.getId())
.deviceCreationTime(device.getCreatedTime())
.metaData(md)
.state(deviceState).build();
} catch (Exception e) {

4
application/src/main/java/org/thingsboard/server/service/state/DeviceStateData.java

@ -30,8 +30,8 @@ class DeviceStateData {
private final TenantId tenantId;
private final DeviceId deviceId;
private final long deviceCreationTime;
private TbMsgMetaData metaData;
private final DeviceState state;
}

7
application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java

@ -216,6 +216,11 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer
@Override
public void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, TbCallback callback) {
onAttributesUpdate(tenantId, entityId, scope, attributes, callback, true);
}
@Override
public void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, TbCallback callback, boolean notifyDevice) {
onLocalSubUpdate(entityId,
s -> {
if (TbSubscriptionType.ATTRIBUTES.equals(s.getType())) {
@ -244,7 +249,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer
deviceStateService.onDeviceInactivityTimeoutUpdate(new DeviceId(entityId.getId()), attribute.getLongValue().orElse(0L));
}
}
} else if (TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope)) {
} else if (TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope) && notifyDevice) {
clusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate(tenantId,
new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, new ArrayList<>(attributes))
, null);

2
application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java

@ -35,4 +35,6 @@ public interface SubscriptionManagerService extends ApplicationListener<Partitio
void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, TbCallback callback);
void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, TbCallback callback, boolean notifyDevice);
}

11
application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java

@ -128,9 +128,14 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio
@Override
public void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, FutureCallback<Void> callback) {
saveAndNotify(tenantId, entityId, scope, attributes, callback, true);
}
@Override
public void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, FutureCallback<Void> callback, boolean notifyDevice) {
ListenableFuture<List<Void>> saveFuture = attrService.save(tenantId, entityId, scope, attributes);
addMainCallback(saveFuture, callback);
addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes));
addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice));
}
@Override
@ -157,11 +162,11 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio
, System.currentTimeMillis())), callback);
}
private void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes) {
private void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, boolean notifyDevice) {
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId);
if (currentPartitions.contains(tpi)) {
if (subscriptionManagerService.isPresent()) {
subscriptionManagerService.get().onAttributesUpdate(tenantId, entityId, scope, attributes, TbCallback.EMPTY);
subscriptionManagerService.get().onAttributesUpdate(tenantId, entityId, scope, attributes, TbCallback.EMPTY, notifyDevice);
} else {
log.warn("Possible misconfiguration because subscriptionManagerService is null!");
}

19
application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java

@ -38,19 +38,14 @@ public abstract class AbstractCleanUpService {
@Value("${spring.datasource.password}")
protected String dbPassword;
protected long executeQuery(Connection conn, String query) {
long removed = 0L;
try {
Statement statement = conn.createStatement();
ResultSet resultSet = statement.executeQuery(query);
getWarnings(statement);
protected long executeQuery(Connection conn, String query) throws SQLException {
try (Statement statement = conn.createStatement(); ResultSet resultSet = statement.executeQuery(query)) {
if (log.isDebugEnabled()) {
getWarnings(statement);
}
resultSet.next();
removed = resultSet.getLong(1);
log.debug("Successfully executed query: {}", query);
} catch (SQLException e) {
log.debug("Failed to execute query: {} due to: {}", query, e.getMessage());
return resultSet.getLong(1);
}
return removed;
}
protected void getWarnings(Statement statement) throws SQLException {
@ -65,6 +60,6 @@ public abstract class AbstractCleanUpService {
}
}
protected abstract void doCleanUp(Connection connection);
protected abstract void doCleanUp(Connection connection) throws SQLException;
}

2
application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java

@ -54,7 +54,7 @@ public class EventsCleanUpService extends AbstractCleanUpService {
}
@Override
protected void doCleanUp(Connection connection) {
protected void doCleanUp(Connection connection) throws SQLException {
long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", " + debugTtl + ", 0);");
log.info("Total events removed by TTL: [{}]", totalEventsRemoved);
}

3
application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java

@ -22,6 +22,7 @@ import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.util.PsqlTsDao;
import java.sql.Connection;
import java.sql.SQLException;
@PsqlTsDao
@Service
@ -32,7 +33,7 @@ public class PsqlTimeseriesCleanUpService extends AbstractTimeseriesCleanUpServi
private String partitionType;
@Override
protected void doCleanUp(Connection connection) {
protected void doCleanUp(Connection connection) throws SQLException {
long totalPartitionsRemoved = executeQuery(connection, "call drop_partitions_by_max_ttl('" + partitionType + "'," + systemTtl + ", 0);");
log.info("Total partitions removed by TTL: [{}]", totalPartitionsRemoved);
long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID_STR + "'," + systemTtl + ", 0);");

3
application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java

@ -21,6 +21,7 @@ import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.util.TimescaleDBTsDao;
import java.sql.Connection;
import java.sql.SQLException;
@TimescaleDBTsDao
@Service
@ -28,7 +29,7 @@ import java.sql.Connection;
public class TimescaleTimeseriesCleanUpService extends AbstractTimeseriesCleanUpService {
@Override
protected void doCleanUp(Connection connection) {
protected void doCleanUp(Connection connection) throws SQLException {
long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID_STR + "'," + systemTtl + ", 0);");
log.info("Total telemetry removed stats by TTL for entities: [{}]", totalEntitiesTelemetryRemoved);
}

2
application/src/main/resources/logback.xml

@ -29,6 +29,8 @@
<!-- <logger name="org.thingsboard.server.service.queue" level="TRACE" />-->
<!-- <logger name="org.thingsboard.server.service.transport" level="TRACE" />-->
<!-- <logger name="org.thingsboard.server.queue.memory.InMemoryStorage" level="DEBUG" />-->
<logger name="com.microsoft.azure.servicebus.primitives.CoreMessageReceiver" level="OFF" />

19
application/src/main/resources/thingsboard.yml

@ -569,6 +569,8 @@ transport:
key_password: "${MQTT_SSL_KEY_PASSWORD:server_key_password}"
# Type of the key store
key_store_type: "${MQTT_SSL_KEY_STORE_TYPE:JKS}"
# Skip certificate validity check for client certificates.
skip_validity_check_for_client_cert: "${MQTT_SSL_SKIP_VALIDITY_CHECK_FOR_CLIENT_CERT:false}"
# Local CoAP transport parameters
coap:
# Enable/disable coap transport protocol.
@ -594,6 +596,10 @@ swagger:
queue:
type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ)
in_memory:
stats:
# For debug lvl
print-interval-ms: "${TB_QUEUE_IN_MEMORY_STATS_PRINT_INTERVAL_MS:60000}"
kafka:
bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}"
acks: "${TB_KAFKA_ACKS:all}"
@ -613,11 +619,11 @@ queue:
security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}"
other:
topic-properties:
rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}"
rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100}"
aws_sqs:
use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}"
access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}"
@ -725,6 +731,7 @@ queue:
retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited
failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages;
pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries;
max-pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:3}"# Max allowed time in seconds for pause between retries.
- name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}"
topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}"
poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}"
@ -740,6 +747,7 @@ queue:
retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited
failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages;
pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries;
max-pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}"# Max allowed time in seconds for pause between retries.
- name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}"
topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}"
poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}"
@ -755,6 +763,7 @@ queue:
retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited
failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages;
pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries;
max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}"# Max allowed time in seconds for pause between retries.
transport:
# For high priority notifications that require minimum latency and processing time
notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}"

1
application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java

@ -173,7 +173,6 @@ public abstract class AbstractControllerTest {
.apply(springSecurity()).build();
}
loginSysAdmin();
Tenant tenant = new Tenant();
tenant.setTitle(TEST_TENANT_NAME);
Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class);

2
application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java

@ -51,7 +51,7 @@ public class TbMsgPackProcessingContextTest {
messages.put(UUID.randomUUID(), new TbProtoQueueMsg<>(UUID.randomUUID(), null));
}
when(strategyMock.getPendingMap()).thenReturn(messages);
TbMsgPackProcessingContext context = new TbMsgPackProcessingContext(strategyMock);
TbMsgPackProcessingContext context = new TbMsgPackProcessingContext("Main", strategyMock);
for (UUID uuid : messages.keySet()) {
for (int i = 0; i < parallelCount; i++) {
executorService.submit(() -> context.onSuccess(uuid));

4
common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeInfo.java

@ -15,12 +15,16 @@
*/
package org.thingsboard.server.common.msg.queue;
import lombok.Getter;
import org.thingsboard.server.common.data.id.RuleNodeId;
public class RuleNodeInfo {
private final String label;
@Getter
private final RuleNodeId ruleNodeId;
public RuleNodeInfo(RuleNodeId id, String ruleChainName, String ruleNodeName) {
this.ruleNodeId = id;
this.label = "[RuleChain: " + ruleChainName + "|RuleNode: " + ruleNodeName + "(" + id + ")]";
}

8
common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java

@ -15,6 +15,8 @@
*/
package org.thingsboard.server.common.msg.queue;
import org.thingsboard.server.common.data.id.RuleNodeId;
public interface TbMsgCallback {
TbMsgCallback EMPTY = new TbMsgCallback() {
@ -34,7 +36,11 @@ public interface TbMsgCallback {
void onFailure(RuleEngineException e);
default void visit(RuleNodeInfo ruleNodeInfo) {
default void onProcessingStart(RuleNodeInfo ruleNodeInfo) {
}
default void onProcessingEnd(RuleNodeId ruleNodeId) {
}
}

10
common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java

@ -37,6 +37,7 @@ public class TbKafkaAdmin implements TbQueueAdmin {
private final AdminClient client;
private final Map<String, String> topicConfigs;
private final Set<String> topics = ConcurrentHashMap.newKeySet();
private final int numPartitions;
private final short replicationFactor;
@ -50,6 +51,13 @@ public class TbKafkaAdmin implements TbQueueAdmin {
log.error("Failed to get all topics.", e);
}
String numPartitionsStr = topicConfigs.get("partitions");
if (numPartitionsStr != null) {
numPartitions = Integer.parseInt(numPartitionsStr);
topicConfigs.remove("partitions");
} else {
numPartitions = 1;
}
replicationFactor = settings.getReplicationFactor();
}
@ -59,7 +67,7 @@ public class TbKafkaAdmin implements TbQueueAdmin {
return;
}
try {
NewTopic newTopic = new NewTopic(topic, 1, replicationFactor).configs(topicConfigs);
NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor).configs(topicConfigs);
createTopic(newTopic).values().get(topic).get();
topics.add(topic);
} catch (ExecutionException ee) {

15
common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java

@ -23,27 +23,21 @@ import java.util.Collections;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@Slf4j
public final class InMemoryStorage {
private static InMemoryStorage instance;
private final ConcurrentHashMap<String, BlockingQueue<TbQueueMsg>> storage;
private static ScheduledExecutorService statExecutor;
private InMemoryStorage() {
storage = new ConcurrentHashMap<>();
statExecutor = Executors.newSingleThreadScheduledExecutor();
statExecutor.scheduleAtFixedRate(this::printStats, 60, 60, TimeUnit.SECONDS);
}
private void printStats() {
public void printStats() {
storage.forEach((topic, queue) -> {
if (queue.size() > 0) {
log.debug("Topic: [{}], Queue size: [{}]", topic, queue.size());
log.debug("[{}] Queue Size [{}]", topic, queue.size());
}
});
}
@ -90,9 +84,4 @@ public final class InMemoryStorage {
storage.clear();
}
public void destroy() {
if (statExecutor != null) {
statExecutor.shutdownNow();
}
}
}

2
common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java

@ -53,6 +53,6 @@ public class InMemoryTbQueueProducer<T extends TbQueueMsg> implements TbQueuePro
@Override
public void stop() {
storage.destroy();
}
}

9
common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java

@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.gen.js.JsInvokeProtos;
@ -28,6 +29,7 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer;
import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
@ -47,6 +49,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
private final TbQueueRuleEngineSettings ruleEngineSettings;
private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final InMemoryStorage storage;
public InMemoryMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings,
TbQueueRuleEngineSettings ruleEngineSettings,
@ -59,6 +62,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
this.ruleEngineSettings = ruleEngineSettings;
this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings;
this.storage = InMemoryStorage.getInstance();
}
@Override
@ -120,4 +124,9 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
return null;
}
@Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}")
private void printInMemoryStats() {
storage.printStats();
}
}

1
common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java

@ -24,5 +24,6 @@ public class TbRuleEngineQueueAckStrategyConfiguration {
private int retries;
private double failurePercentage;
private long pauseBetweenRetries;
private long maxPauseBetweenRetries;
}

5
common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java

@ -87,8 +87,9 @@ public class TbAwsSqsProducerTemplate<T extends TbQueueMsg> implements TbQueuePr
sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName()));
sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg)));
sendMsgRequest.withMessageGroupId(tpi.getTopic());
sendMsgRequest.withMessageDeduplicationId(UUID.randomUUID().toString());
String sqsMsgId = UUID.randomUUID().toString();
sendMsgRequest.withMessageGroupId(sqsMsgId);
sendMsgRequest.withMessageDeduplicationId(sqsMsgId);
ListenableFuture<SendMessageResult> future = producerExecutor.submit(() -> sqsClient.sendMessage(sendMsgRequest));

4
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java

@ -46,6 +46,10 @@ public class MqttTransportContext extends TransportContext {
@Value("${transport.mqtt.netty.max_payload_size}")
private Integer maxPayloadSize;
@Getter
@Value("${transport.mqtt.netty.skip_validity_check_for_client_cert:false}")
private boolean skipValidityCheckForClientCert;
@Getter
@Setter
private SslHandler sslHandler;

17
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java

@ -115,7 +115,13 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
log.trace("[{}] Processing msg: {}", sessionId, msg);
try {
if (msg instanceof MqttMessage) {
processMqttMsg(ctx, (MqttMessage) msg);
MqttMessage message = (MqttMessage) msg;
if (message.decoderResult().isSuccess()) {
processMqttMsg(ctx, message);
} else {
log.error("[{}] Message processing failed: {}", sessionId, message.decoderResult().cause().getMessage());
ctx.close();
}
} else {
ctx.close();
}
@ -342,7 +348,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
private MqttMessage createUnSubAckMessage(int msgId) {
MqttFixedHeader mqttFixedHeader =
new MqttFixedHeader(UNSUBACK, false, AT_LEAST_ONCE, false, 0);
new MqttFixedHeader(UNSUBACK, false, AT_MOST_ONCE, false, 0);
MqttMessageIdVariableHeader mqttMessageIdVariableHeader = MqttMessageIdVariableHeader.from(msgId);
return new MqttMessage(mqttFixedHeader, mqttMessageIdVariableHeader);
}
@ -383,6 +389,9 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
private void processX509CertConnect(ChannelHandlerContext ctx, X509Certificate cert) {
try {
if(!context.isSkipValidityCheckForClientCert()){
cert.checkValidity();
}
String strCert = SslUtil.getX509CertificateString(cert);
String sha3Hash = EncryptionUtil.getSha3Hash(strCert);
transportService.process(ValidateDeviceX509CertRequestMsg.newBuilder().setHash(sha3Hash).build(),
@ -445,7 +454,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
private static MqttSubAckMessage createSubAckMessage(Integer msgId, List<Integer> grantedQoSList) {
MqttFixedHeader mqttFixedHeader =
new MqttFixedHeader(SUBACK, false, AT_LEAST_ONCE, false, 0);
new MqttFixedHeader(SUBACK, false, AT_MOST_ONCE, false, 0);
MqttMessageIdVariableHeader mqttMessageIdVariableHeader = MqttMessageIdVariableHeader.from(msgId);
MqttSubAckPayload mqttSubAckPayload = new MqttSubAckPayload(grantedQoSList);
return new MqttSubAckMessage(mqttFixedHeader, mqttMessageIdVariableHeader, mqttSubAckPayload);
@ -457,7 +466,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
public static MqttPubAckMessage createMqttPubAckMsg(int requestId) {
MqttFixedHeader mqttFixedHeader =
new MqttFixedHeader(PUBACK, false, AT_LEAST_ONCE, false, 0);
new MqttFixedHeader(PUBACK, false, AT_MOST_ONCE, false, 0);
MqttMessageIdVariableHeader mqttMsgIdVariableHeader =
MqttMessageIdVariableHeader.from(requestId);
return new MqttPubAckMessage(mqttFixedHeader, mqttMsgIdVariableHeader);

5
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java

@ -298,6 +298,7 @@ public class DefaultTransportService implements TransportService {
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("deviceName", sessionInfo.getDeviceName());
metaData.putValue("deviceType", sessionInfo.getDeviceType());
metaData.putValue("notifyDevice", "false");
TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, metaData, gson.toJson(json));
sendToRuleEngine(tenantId, tbMsg, new TransportTbQueueCallback(callback));
}
@ -417,8 +418,8 @@ public class DefaultTransportService implements TransportService {
sessions.forEach((uuid, sessionMD) -> {
long lastActivityTime = sessionMD.getLastActivityTime();
TransportProtos.SessionInfoProto sessionInfo = sessionMD.getSessionInfo();
if (sessionInfo.getGwSessionIdMSB() > 0 &&
sessionInfo.getGwSessionIdLSB() > 0) {
if (sessionInfo.getGwSessionIdMSB() != 0 &&
sessionInfo.getGwSessionIdLSB() != 0) {
SessionMetaData gwMetaData = sessions.get(new UUID(sessionInfo.getGwSessionIdMSB(), sessionInfo.getGwSessionIdLSB()));
if (gwMetaData != null) {
lastActivityTime = Math.max(gwMetaData.getLastActivityTime(), lastActivityTime);

2
dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java

@ -388,7 +388,7 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe
if (StringUtils.isEmpty(device.getType())) {
throw new DataValidationException("Device type should be specified!");
}
if (StringUtils.isEmpty(device.getName())) {
if (StringUtils.isEmpty(device.getName()) || device.getName().trim().length() == 0) {
throw new DataValidationException("Device name should be specified!");
}
if (device.getTenantId() == null) {

17
dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.dao.event;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@ -28,6 +29,7 @@ import org.thingsboard.server.common.data.page.TimePageLink;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.service.DataValidator;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Optional;
@ -35,6 +37,8 @@ import java.util.Optional;
@Slf4j
public class BaseEventService implements EventService {
private static final int MAX_DEBUG_EVENT_SYMBOLS = 4 * 1024;
@Autowired
public EventDao eventDao;
@ -47,6 +51,7 @@ public class BaseEventService implements EventService {
@Override
public ListenableFuture<Event> saveAsync(Event event) {
eventValidator.validate(event, Event::getTenantId);
checkAndTruncateDebugEvent(event);
return eventDao.saveAsync(event);
}
@ -56,9 +61,21 @@ public class BaseEventService implements EventService {
if (StringUtils.isEmpty(event.getUid())) {
throw new DataValidationException("Event uid should be specified!.");
}
checkAndTruncateDebugEvent(event);
return eventDao.saveIfNotExists(event);
}
private void checkAndTruncateDebugEvent(Event event) {
if (event.getType().startsWith("DEBUG") && event.getBody() != null && event.getBody().has("data")) {
String dataStr = event.getBody().get("data").asText();
int length = dataStr.length();
if (length > MAX_DEBUG_EVENT_SYMBOLS) {
((ObjectNode) event.getBody()).put("data", dataStr.substring(0, MAX_DEBUG_EVENT_SYMBOLS) + "...[truncated " + (length - MAX_DEBUG_EVENT_SYMBOLS) + " symbols]");
log.trace("[{}] Event was truncated: {}", event.getId(), dataStr);
}
}
}
@Override
public Optional<Event> findEvent(TenantId tenantId, EntityId entityId, String eventType, String eventUid) {
if (tenantId == null) {

33
dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java

@ -17,6 +17,7 @@ package org.thingsboard.server.dao.rule;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -45,10 +46,11 @@ import org.thingsboard.server.dao.tenant.TenantDao;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
/**
* Created by igor on 3/12/18.
@ -123,6 +125,10 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC
return null;
}
if (CollectionUtils.isNotEmpty(ruleChainMetaData.getConnections())) {
validateCircles(ruleChainMetaData.getConnections());
}
List<RuleNode> nodes = ruleChainMetaData.getNodes();
List<RuleNode> toAddOrUpdate = new ArrayList<>();
List<RuleNode> toDelete = new ArrayList<>();
@ -205,6 +211,31 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC
return loadRuleChainMetaData(tenantId, ruleChainMetaData.getRuleChainId());
}
private void validateCircles(List<NodeConnectionInfo> connectionInfos) {
Map<Integer, Set<Integer>> connectionsMap = new HashMap<>();
for (NodeConnectionInfo nodeConnection : connectionInfos) {
if (nodeConnection.getFromIndex() == nodeConnection.getToIndex()) {
throw new DataValidationException("Can't create the relation to yourself.");
}
connectionsMap
.computeIfAbsent(nodeConnection.getFromIndex(), from -> new HashSet<>())
.add(nodeConnection.getToIndex());
}
connectionsMap.keySet().forEach(key -> validateCircles(key, connectionsMap.get(key), connectionsMap));
}
private void validateCircles(int from, Set<Integer> toList, Map<Integer, Set<Integer>> connectionsMap) {
if (toList == null) {
return;
}
for (Integer to : toList) {
if (from == to) {
throw new DataValidationException("Can't create circling relations in rule chain.");
}
validateCircles(from, connectionsMap.get(to), connectionsMap);
}
}
@Override
public RuleChainMetaData loadRuleChainMetaData(TenantId tenantId, RuleChainId ruleChainId) {
Validator.validateId(ruleChainId, "Incorrect rule chain id.");

1
dao/src/main/resources/sql/schema-ts-psql.sql

@ -105,6 +105,7 @@ BEGIN
AND tablename like 'ts_kv_' || '%'
AND tablename != 'ts_kv_latest'
AND tablename != 'ts_kv_dictionary'
AND tablename != 'ts_kv_indefinite'
LOOP
IF partition != partition_by_max_ttl_date THEN
IF partition_year IS NOT NULL THEN

90
dao/src/test/java/org/thingsboard/server/dao/service/BaseRuleChainServiceTest.java

@ -317,6 +317,16 @@ public abstract class BaseRuleChainServiceTest extends AbstractServiceTest {
ruleChainService.deleteRuleChainById(tenantId, savedRuleChainMetaData.getRuleChainId());
}
@Test(expected = DataValidationException.class)
public void testUpdateRuleChainMetaDataWithCirclingRelation() throws Exception {
ruleChainService.saveRuleChainMetaData(tenantId, createRuleChainMetadataWithCirclingRelation());
}
@Test(expected = DataValidationException.class)
public void testUpdateRuleChainMetaDataWithCirclingRelation2() throws Exception {
ruleChainService.saveRuleChainMetaData(tenantId, createRuleChainMetadataWithCirclingRelation2());
}
private RuleChainMetaData createRuleChainMetadata() throws Exception {
RuleChain ruleChain = new RuleChain();
ruleChain.setName("My RuleChain");
@ -357,5 +367,85 @@ public abstract class BaseRuleChainServiceTest extends AbstractServiceTest {
return ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData);
}
private RuleChainMetaData createRuleChainMetadataWithCirclingRelation() throws Exception {
RuleChain ruleChain = new RuleChain();
ruleChain.setName("My RuleChain");
ruleChain.setTenantId(tenantId);
RuleChain savedRuleChain = ruleChainService.saveRuleChain(ruleChain);
RuleChainMetaData ruleChainMetaData = new RuleChainMetaData();
ruleChainMetaData.setRuleChainId(savedRuleChain.getId());
ObjectMapper mapper = new ObjectMapper();
RuleNode ruleNode1 = new RuleNode();
ruleNode1.setName("name1");
ruleNode1.setType("type1");
ruleNode1.setConfiguration(mapper.readTree("\"key1\": \"val1\""));
RuleNode ruleNode2 = new RuleNode();
ruleNode2.setName("name2");
ruleNode2.setType("type2");
ruleNode2.setConfiguration(mapper.readTree("\"key2\": \"val2\""));
RuleNode ruleNode3 = new RuleNode();
ruleNode3.setName("name3");
ruleNode3.setType("type3");
ruleNode3.setConfiguration(mapper.readTree("\"key3\": \"val3\""));
List<RuleNode> ruleNodes = new ArrayList<>();
ruleNodes.add(ruleNode1);
ruleNodes.add(ruleNode2);
ruleNodes.add(ruleNode3);
ruleChainMetaData.setFirstNodeIndex(0);
ruleChainMetaData.setNodes(ruleNodes);
ruleChainMetaData.addConnectionInfo(0,1,"success");
ruleChainMetaData.addConnectionInfo(0,2,"fail");
ruleChainMetaData.addConnectionInfo(1,2,"success");
ruleChainMetaData.addConnectionInfo(2,2,"success");
return ruleChainMetaData;
}
private RuleChainMetaData createRuleChainMetadataWithCirclingRelation2() throws Exception {
RuleChain ruleChain = new RuleChain();
ruleChain.setName("My RuleChain");
ruleChain.setTenantId(tenantId);
RuleChain savedRuleChain = ruleChainService.saveRuleChain(ruleChain);
RuleChainMetaData ruleChainMetaData = new RuleChainMetaData();
ruleChainMetaData.setRuleChainId(savedRuleChain.getId());
ObjectMapper mapper = new ObjectMapper();
RuleNode ruleNode1 = new RuleNode();
ruleNode1.setName("name1");
ruleNode1.setType("type1");
ruleNode1.setConfiguration(mapper.readTree("\"key1\": \"val1\""));
RuleNode ruleNode2 = new RuleNode();
ruleNode2.setName("name2");
ruleNode2.setType("type2");
ruleNode2.setConfiguration(mapper.readTree("\"key2\": \"val2\""));
RuleNode ruleNode3 = new RuleNode();
ruleNode3.setName("name3");
ruleNode3.setType("type3");
ruleNode3.setConfiguration(mapper.readTree("\"key3\": \"val3\""));
List<RuleNode> ruleNodes = new ArrayList<>();
ruleNodes.add(ruleNode1);
ruleNodes.add(ruleNode2);
ruleNodes.add(ruleNode3);
ruleChainMetaData.setFirstNodeIndex(0);
ruleChainMetaData.setNodes(ruleNodes);
ruleChainMetaData.addConnectionInfo(0,1,"success");
ruleChainMetaData.addConnectionInfo(0,2,"fail");
ruleChainMetaData.addConnectionInfo(1,2,"success");
ruleChainMetaData.addConnectionInfo(2,0,"success");
return ruleChainMetaData;
}
}

2
dao/src/test/resources/sql-test.properties

@ -12,7 +12,7 @@ spring.jpa.database-platform=org.hibernate.dialect.HSQLDialect
spring.datasource.username=sa
spring.datasource.password=
spring.datasource.url=jdbc:hsqldb:file:/tmp/testDb;sql.enforce_size=false
spring.datasource.url=jdbc:hsqldb:file:target/tmp/testDb;sql.enforce_size=false
spring.datasource.driverClassName=org.hsqldb.jdbc.JDBCDriver
spring.datasource.hikari.maximumPoolSize = 50

2
msa/js-executor/config/default.yml

@ -25,7 +25,7 @@ kafka:
# Kafka Bootstrap Servers
servers: "localhost:9092"
replication_factor: "1"
topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600"
topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100"
use_confluent_cloud: false
confluent:
sasl:

2599
msa/js-executor/package-lock.json

File diff suppressed because it is too large

4
msa/js-executor/package.json

@ -1,7 +1,7 @@
{
"name": "thingsboard-js-executor",
"private": true,
"version": "2.5.4",
"version": "2.5.5",
"description": "ThingsBoard JavaScript Executor Microservice",
"main": "server.js",
"bin": "server.js",
@ -14,7 +14,7 @@
"dependencies": {
"config": "^3.2.2",
"js-yaml": "^3.12.0",
"kafkajs": "^1.12.0",
"kafkajs": "^1.14.0",
"@google-cloud/pubsub": "^1.7.1",
"aws-sdk": "^2.663.0",
"amqplib": "^0.5.5",

6
msa/js-executor/queue/awsSqsTemplate.js

@ -52,11 +52,13 @@ function AwsSqsProducer() {
queueUrls.set(responseTopic, responseQueueUrl);
}
let msgId = uuid();
let params = {
MessageBody: msgBody,
QueueUrl: responseQueueUrl,
MessageGroupId: 'js_eval',
MessageDeduplicationId: uuid()
MessageGroupId: msgId,
MessageDeduplicationId: msgId
};
return new Promise((resolve, reject) => {

33
msa/js-executor/queue/kafkaTemplate.js

@ -27,20 +27,10 @@ let kafkaAdmin;
let consumer;
let producer;
const topics = [];
const configEntries = [];
function KafkaProducer() {
this.send = async (responseTopic, scriptId, rawResponse, headers) => {
if (!topics.includes(responseTopic)) {
let createResponseTopicResult = await createTopic(responseTopic);
topics.push(responseTopic);
if (createResponseTopicResult) {
logger.info('Created new topic: %s', requestTopic);
}
}
return producer.send(
{
topic: responseTopic,
@ -88,10 +78,24 @@ function KafkaProducer() {
kafkaAdmin = kafkaClient.admin();
await kafkaAdmin.connect();
let createRequestTopicResult = await createTopic(requestTopic);
let partitions = 1;
if (createRequestTopicResult) {
logger.info('Created new topic: %s', requestTopic);
for (let i = 0; i < configEntries.length; i++) {
let param = configEntries[i];
if (param.name === 'partitions') {
partitions = param.value;
configEntries.splice(i, 1);
break;
}
}
let topics = await kafkaAdmin.listTopics();
if (!topics.includes(requestTopic)) {
let createRequestTopicResult = await createTopic(requestTopic, partitions);
if (createRequestTopicResult) {
logger.info('Created new topic: %s', requestTopic);
}
}
consumer = kafkaClient.consumer({groupId: 'js-executor-group'});
@ -121,10 +125,11 @@ function KafkaProducer() {
}
})();
function createTopic(topic) {
function createTopic(topic, partitions) {
return kafkaAdmin.createTopics({
topics: [{
topic: topic,
numPartitions: partitions,
replicationFactor: replicationFactor,
configEntries: configEntries
}]

2
msa/web-ui/package.json

@ -1,7 +1,7 @@
{
"name": "thingsboard-web-ui",
"private": true,
"version": "2.5.4",
"version": "2.5.5",
"description": "ThingsBoard Web UI Microservice",
"main": "server.js",
"bin": "server.js",

33
pom.xml

@ -36,12 +36,11 @@
<pkg.implementationTitle>${project.name}</pkg.implementationTitle>
<pkg.unixLogFolder>/var/log/${pkg.name}</pkg.unixLogFolder>
<pkg.installFolder>/usr/share/${pkg.name}</pkg.installFolder>
<spring-boot.version>2.2.6.RELEASE</spring-boot.version>
<spring-oauth2.version>2.1.2.RELEASE</spring-oauth2.version>
<spring.version>5.2.6.RELEASE</spring.version>
<spring-security.version>5.2.3.RELEASE</spring-security.version>
<spring-data-redis.version>2.2.4.RELEASE</spring-data-redis.version>
<jedis.version>3.1.0</jedis.version>
<spring-boot.version>2.3.5.RELEASE</spring-boot.version>
<spring.version>5.2.10.RELEASE</spring.version>
<spring-security.version>5.4.1</spring-security.version>
<spring-data-redis.version>2.4.1</spring-data-redis.version>
<jedis.version>3.3.0</jedis.version>
<jjwt.version>0.7.0</jjwt.version>
<json-path.version>2.2.0</json-path.version>
<junit.version>4.12</junit.version>
@ -51,15 +50,16 @@
<rat.version>0.10</rat.version>
<cassandra.version>3.6.0</cassandra.version>
<cassandra-unit.version>3.5.0.1</cassandra-unit.version>
<cassandra-all.version>3.11.9</cassandra-all.version>
<takari-cpsuite.version>1.2.7</takari-cpsuite.version>
<guava.version>28.2-jre</guava.version>
<caffeine.version>2.6.1</caffeine.version>
<commons-lang3.version>3.4</commons-lang3.version>
<commons-io.version>2.5</commons-io.version>
<commons-csv.version>1.4</commons-csv.version>
<jackson.version>2.10.2</jackson.version>
<jackson-annotations.version>2.10.2</jackson-annotations.version>
<jackson-core.version>2.10.2</jackson-core.version>
<jackson.version>2.11.3</jackson.version>
<jackson-annotations.version>2.11.3</jackson-annotations.version>
<jackson-core.version>2.11.3</jackson-core.version>
<json-schema-validator.version>2.2.6</json-schema-validator.version>
<californium.version>1.0.2</californium.version>
<gson.version>2.6.2</gson.version>
@ -72,7 +72,7 @@
<grpc.version>1.22.1</grpc.version>
<lombok.version>1.16.18</lombok.version>
<paho.client.version>1.1.0</paho.client.version>
<netty.version>4.1.49.Final</netty.version>
<netty.version>4.1.53.Final</netty.version>
<os-maven-plugin.version>1.5.0</os-maven-plugin.version>
<rabbitmq.version>4.8.0</rabbitmq.version>
<surfire.version>2.19.1</surfire.version>
@ -96,7 +96,7 @@
<bucket4j.version>4.1.1</bucket4j.version>
<fst.version>2.57</fst.version>
<antlr.version>2.7.7</antlr.version>
<snakeyaml.version>1.25</snakeyaml.version>
<snakeyaml.version>1.27</snakeyaml.version>
<struts.version>1.3.10</struts.version>
<amazonaws.sqs.version>1.11.747</amazonaws.sqs.version>
<pubsub.client.version>1.105.0</pubsub.client.version>
@ -725,6 +725,7 @@
<exclude>**/*.proto.js</exclude>
<exclude>docker/haproxy/**</exclude>
<exclude>docker/tb-node/**</exclude>
<exclude>.run/**</exclude>
</excludes>
<mapping>
<proto>JAVADOC_STYLE</proto>
@ -867,11 +868,6 @@
<artifactId>spring-boot-starter-security</artifactId>
<version>${spring-boot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-oauth2</artifactId>
<version>${spring-oauth2.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-oauth2-client</artifactId>
@ -1214,6 +1210,11 @@
<version>${cassandra-unit.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.cassandra</groupId>
<artifactId>cassandra-all</artifactId>
<version>${cassandra-all.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>

2
rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java

@ -36,6 +36,8 @@ public interface RuleEngineTelemetryService {
void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, FutureCallback<Void> callback);
void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, FutureCallback<Void> callback, boolean notifyDevice);
void saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, long value, FutureCallback<Void> callback);
void saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, String value, FutureCallback<Void> callback);

2
rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java

@ -141,7 +141,7 @@ public interface TbContext {
TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId);
// TODO: Does this changes the message?
TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId);
TbMsg alarmActionMsg(Alarm alarm, RuleNodeId ruleNodeId, String action);
/*
*

5
rule-engine/rule-engine-components/pom.xml

@ -120,6 +120,11 @@
<groupId>org.locationtech.jts</groupId>
<artifactId>jts-core</artifactId>
</dependency>
<dependency>
<groupId>com.sun.mail</groupId>
<artifactId>javax.mail</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>

15
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java

@ -25,6 +25,7 @@ import org.thingsboard.rule.engine.api.TbContext;
import org.thingsboard.rule.engine.api.TbNode;
import org.thingsboard.rule.engine.api.TbNodeConfiguration;
import org.thingsboard.rule.engine.api.TbNodeException;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.alarm.Alarm;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgMetaData;
@ -61,13 +62,11 @@ public abstract class TbAbstractAlarmNode<C extends TbAbstractAlarmNodeConfigura
if (alarmResult.alarm == null) {
ctx.tellNext(msg, "False");
} else if (alarmResult.isCreated) {
ctx.enqueue(ctx.alarmCreatedMsg(alarmResult.alarm, ctx.getSelfId()),
() -> ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Created"),
throwable -> ctx.tellFailure(toAlarmMsg(ctx, alarmResult, msg), throwable));
tellNext(ctx, msg, alarmResult, DataConstants.ENTITY_CREATED, "Created");
} else if (alarmResult.isUpdated) {
ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Updated");
tellNext(ctx, msg, alarmResult, DataConstants.ENTITY_UPDATED, "Updated");
} else if (alarmResult.isCleared) {
ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Cleared");
tellNext(ctx, msg, alarmResult, DataConstants.ALARM_CLEAR, "Cleared");
} else {
ctx.tellSuccess(msg);
}
@ -126,4 +125,10 @@ public abstract class TbAbstractAlarmNode<C extends TbAbstractAlarmNodeConfigura
this.alarm = alarm;
}
}
private void tellNext(TbContext ctx, TbMsg msg, AlarmResult alarmResult, String alarmAction, String alarmResultMsgType) {
ctx.enqueue(ctx.alarmActionMsg(alarmResult.alarm, ctx.getSelfId(), alarmAction),
() -> ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), alarmResultMsgType),
throwable -> ctx.tellFailure(toAlarmMsg(ctx, alarmResult, msg), throwable));
}
}

17
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java

@ -17,15 +17,13 @@ package org.thingsboard.rule.engine.telemetry;
import com.google.gson.JsonParser;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.thingsboard.rule.engine.api.RuleNode;
import org.thingsboard.rule.engine.api.TbContext;
import org.thingsboard.rule.engine.api.TbNode;
import org.thingsboard.rule.engine.api.TbNodeConfiguration;
import org.thingsboard.rule.engine.api.TbNodeException;
import org.thingsboard.rule.engine.api.util.TbNodeUtils;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.plugin.ComponentType;
import org.thingsboard.server.common.msg.TbMsg;
@ -53,6 +51,9 @@ public class TbMsgAttributesNode implements TbNode {
@Override
public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException {
this.config = TbNodeUtils.convert(configuration, TbMsgAttributesNodeConfiguration.class);
if (config.getNotifyDevice() == null) {
config.setNotifyDevice(true);
}
}
@Override
@ -63,7 +64,15 @@ public class TbMsgAttributesNode implements TbNode {
}
String src = msg.getData();
Set<AttributeKvEntry> attributes = JsonConverter.convertToAttributes(new JsonParser().parse(src));
ctx.getTelemetryService().saveAndNotify(ctx.getTenantId(), msg.getOriginator(), config.getScope(), new ArrayList<>(attributes), new TelemetryNodeCallback(ctx, msg));
String notifyDeviceStr = msg.getMetaData().getValue("notifyDevice");
ctx.getTelemetryService().saveAndNotify(
ctx.getTenantId(),
msg.getOriginator(),
config.getScope(),
new ArrayList<>(attributes),
new TelemetryNodeCallback(ctx, msg),
config.getNotifyDevice() || StringUtils.isEmpty(notifyDeviceStr) || Boolean.parseBoolean(notifyDeviceStr)
);
}
@Override

3
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeConfiguration.java

@ -24,10 +24,13 @@ public class TbMsgAttributesNodeConfiguration implements NodeConfiguration<TbMsg
private String scope;
private Boolean notifyDevice;
@Override
public TbMsgAttributesNodeConfiguration defaultConfiguration() {
TbMsgAttributesNodeConfiguration configuration = new TbMsgAttributesNodeConfiguration();
configuration.setScope(DataConstants.SERVER_SCOPE);
configuration.setNotifyDevice(false);
return configuration;
}
}

6
rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js

File diff suppressed because one or more lines are too long

7
rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java

@ -34,6 +34,7 @@ import org.thingsboard.rule.engine.api.ScriptEngine;
import org.thingsboard.rule.engine.api.TbContext;
import org.thingsboard.rule.engine.api.TbNodeConfiguration;
import org.thingsboard.rule.engine.api.TbNodeException;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.alarm.Alarm;
import org.thingsboard.server.common.data.id.AlarmId;
import org.thingsboard.server.common.data.id.DeviceId;
@ -249,6 +250,8 @@ public class TbAlarmNodeTest {
node.onMsg(ctx, msg);
verify(ctx).enqueue(any(), successCaptor.capture(), failureCaptor.capture());
successCaptor.getValue().run();
verify(ctx).tellNext(any(), eq("Updated"));
ArgumentCaptor<TbMsg> msgCaptor = ArgumentCaptor.forClass(TbMsg.class);
@ -297,6 +300,8 @@ public class TbAlarmNodeTest {
node.onMsg(ctx, msg);
verify(ctx).enqueue(any(), successCaptor.capture(), failureCaptor.capture());
successCaptor.getValue().run();
verify(ctx).tellNext(any(), eq("Cleared"));
ArgumentCaptor<TbMsg> msgCaptor = ArgumentCaptor.forClass(TbMsg.class);
@ -345,6 +350,8 @@ public class TbAlarmNodeTest {
node.onMsg(ctx, msg);
verify(ctx).enqueue(any(), successCaptor.capture(), failureCaptor.capture());
successCaptor.getValue().run();
verify(ctx).tellNext(any(), eq("Cleared"));
ArgumentCaptor<TbMsg> msgCaptor = ArgumentCaptor.forClass(TbMsg.class);

1
tools/pom.xml

@ -54,7 +54,6 @@
<dependency>
<groupId>org.apache.cassandra</groupId>
<artifactId>cassandra-all</artifactId>
<version>3.11.6</version>
</dependency>
<dependency>
<groupId>com.datastax.cassandra</groupId>

10
transport/coap/src/main/resources/tb-coap-transport.yml

@ -77,11 +77,11 @@ queue:
security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}"
other:
topic-properties:
rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}"
rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100}"
aws_sqs:
use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}"
access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}"

10
transport/http/src/main/resources/tb-http-transport.yml

@ -70,11 +70,11 @@ queue:
security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}"
other:
topic-properties:
rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}"
rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100}"
aws_sqs:
use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}"
access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}"

12
transport/mqtt/src/main/resources/tb-mqtt-transport.yml

@ -67,6 +67,8 @@ transport:
key_password: "${MQTT_SSL_KEY_PASSWORD:server_key_password}"
# Type of the key store
key_store_type: "${MQTT_SSL_KEY_STORE_TYPE:JKS}"
# Skip certificate validity check for client certificates.
skip_validity_check_for_client_cert: "${MQTT_SSL_SKIP_VALIDITY_CHECK_FOR_CLIENT_CERT:false}"
sessions:
inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}"
report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:30000}"
@ -98,11 +100,11 @@ queue:
security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}"
other:
topic-properties:
rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}"
rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100}"
aws_sqs:
use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}"
access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}"

2
ui/package.json

@ -1,7 +1,7 @@
{
"name": "thingsboard",
"private": true,
"version": "2.5.4",
"version": "2.5.5",
"description": "ThingsBoard UI",
"licenses": [
{

8
ui/src/app/admin/general-settings.tpl.html

@ -34,6 +34,14 @@
<div translate ng-message="required">admin.base-url-required</div>
</div>
</md-input-container>
<md-checkbox class="md-block"
aria-label="{{ 'admin.prohibit-different-url' | translate }}"
ng-model="vm.settings.jsonValue.prohibitDifferentUrl">
{{ 'admin.prohibit-different-url' | translate }}
</md-checkbox>
<div translate class="tb-hint">
admin.prohibit-different-url-hint
</div>
<div layout="row" layout-align="end center" width="100%" layout-wrap>
<md-button ng-disabled="$root.loading || vm.settingsForm.$invalid || !vm.settingsForm.$dirty" type="submit" class="md-raised md-primary">{{'action.save' | translate}}</md-button>
</div>

2
ui/src/app/locale/locale.constant-en_US.json

@ -74,6 +74,8 @@
"test-mail-sent": "Test mail was successfully sent!",
"base-url": "Base URL",
"base-url-required": "Base URL is required.",
"prohibit-different-url": "Prohibit to use hostname from the client request headers",
"prohibit-different-url-hint": "This setting should be enabled for production environments. May cause security issues when disabled",
"mail-from": "Mail From",
"mail-from-required": "Mail From is required.",
"smtp-protocol": "SMTP protocol",

17
ui/src/app/widget/lib/rpc/knob.directive.js

@ -50,6 +50,7 @@ function KnobController($element, $scope, $document) {
vm.value = 0;
vm.error = '';
vm.newValue = 0;
var knob = angular.element('.knob', $element),
knobContainer = angular.element('#knob-container', $element),
@ -145,9 +146,11 @@ function KnobController($element, $scope, $document) {
turn(degreeToRatio(currentDeg));
rotation = currentDeg;
startDeg = -1;
rpcUpdateValue(vm.newValue);
});
knob.on('mousedown touchstart', (e) => {
moving = false;
e.preventDefault();
var offset = knob.offset();
var center = {
@ -158,7 +161,7 @@ function KnobController($element, $scope, $document) {
var a, b, deg, tmp,
rad2deg = 180/Math.PI;
knob.on('mousemove.rem touchmove.rem', (e) => {
$document.on('mousemove.rem touchmove.rem', (e) => {
moving = true;
e = (e.originalEvent.touches) ? e.originalEvent.touches[0] : e;
@ -209,6 +212,9 @@ function KnobController($element, $scope, $document) {
});
$document.on('mouseup.rem touchend.rem',() => {
if(moving) {
rpcUpdateValue(vm.newValue);
}
knob.off('.rem');
$document.off('.rem');
rotation = currentDeg;
@ -269,12 +275,12 @@ function KnobController($element, $scope, $document) {
}
function turn(ratio) {
var value = (vm.minValue + (vm.maxValue - vm.minValue)*ratio).toFixed(vm.ctx.decimals);
if (canvasBar.value != value) {
canvasBar.value = value;
vm.newValue = (vm.minValue + (vm.maxValue - vm.minValue)*ratio).toFixed(vm.ctx.decimals);
if (canvasBar.value != vm.newValue) {
canvasBar.value = vm.newValue;
}
updateColor(canvasBar.getValueColor());
onValue(value);
onValue(vm.newValue);
}
function setValue(value) {
@ -303,7 +309,6 @@ function KnobController($element, $scope, $document) {
$scope.$applyAsync(() => {
vm.value = formatValue(value);
checkValueSize();
rpcUpdateValue(value);
});
}

Loading…
Cancel
Save