Browse Source

Merge remote-tracking branch 'upstream/develop/2.5.3' into feature/edge

pull/2436/head
Volodymyr Babak 6 years ago
parent
commit
2b3e195c20
  1. 4
      application/pom.xml
  2. 22
      application/src/main/data/certs/azure/BaltimoreCyberTrustRoot.crt.pem
  3. 3
      application/src/main/java/org/thingsboard/server/ThingsboardInstallApplication.java
  4. 15
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  5. 17
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
  6. 2
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java
  7. 5
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java
  8. 19
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  9. 34
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
  10. 125
      application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java
  11. 8
      application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java
  12. 14
      application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java
  13. 95
      application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java
  14. 4
      application/src/main/java/org/thingsboard/server/service/security/model/token/RawAccessJwtToken.java
  15. 84
      application/src/main/java/org/thingsboard/server/service/stats/DefaultJsInvokeStats.java
  16. 1
      application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java
  17. 12
      application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java
  18. 1
      application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java
  19. 2
      application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java
  20. 2
      application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java
  21. 16
      application/src/main/resources/thingsboard.yml
  22. 44
      common/actor/src/main/java/org/thingsboard/server/actors/JsInvokeStats.java
  23. 4
      common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsDao.java
  24. 1
      common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java
  25. 31
      common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeInfo.java
  26. 3
      common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java
  27. 1
      common/pom.xml
  28. 5
      common/queue/pom.xml
  29. 2
      common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java
  30. 17
      common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java
  31. 10
      common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java
  32. 92
      common/stats/pom.xml
  33. 48
      common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultCounter.java
  34. 65
      common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultMessagesStats.java
  35. 91
      common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java
  36. 44
      common/stats/src/main/java/org/thingsboard/server/common/stats/MessagesStats.java
  37. 33
      common/stats/src/main/java/org/thingsboard/server/common/stats/StatsCounter.java
  38. 26
      common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java
  39. 30
      common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java
  40. 4
      common/transport/transport-api/pom.xml
  41. 51
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java
  42. 99
      common/util/src/main/java/org/thingsboard/common/util/AzureIotHubUtil.java
  43. 4
      dao/pom.xml
  44. 77
      dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateExecutor.java
  45. 4
      dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java
  46. 28
      dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueue.java
  47. 3
      dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueueParams.java
  48. 56
      dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueueWrapper.java
  49. 2
      dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlQueue.java
  50. 17
      dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java
  51. 13
      dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java
  52. 39
      dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java
  53. 26
      dao/src/main/java/org/thingsboard/server/dao/sqlts/TsKey.java
  54. 16
      dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java
  55. 44
      dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java
  56. 90
      dao/src/main/java/org/thingsboard/server/dao/util/BufferedRateExecutorStats.java
  57. 21
      pom.xml
  58. 2
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java
  59. 2
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCreateAlarmNode.java
  60. 2
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java
  61. 33
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java
  62. 91
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/azure/AzureIotHubSasCredentials.java
  63. 86
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/azure/TbAzureIotHubNode.java
  64. 40
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/azure/TbAzureIotHubNodeConfiguration.java
  65. 2
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/credentials/CertPemClientCredentials.java
  66. 2
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/credentials/MqttClientCredentials.java
  67. 10
      rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js
  68. 28
      transport/coap/src/main/resources/tb-coap-transport.yml
  69. 16
      transport/http/src/main/resources/tb-http-transport.yml
  70. 27
      transport/mqtt/src/main/resources/tb-mqtt-transport.yml
  71. 2
      ui/src/app/api/time.service.js

4
application/pom.xml

@ -97,6 +97,10 @@
<groupId>org.thingsboard.common</groupId>
<artifactId>queue</artifactId>
</dependency>
<dependency>
<groupId>org.thingsboard.common</groupId>
<artifactId>stats</artifactId>
</dependency>
<dependency>
<groupId>org.thingsboard.common</groupId>
<artifactId>edge-api</artifactId>

22
application/src/main/data/certs/azure/BaltimoreCyberTrustRoot.crt.pem

@ -0,0 +1,22 @@
-----BEGIN CERTIFICATE-----
MIIDdzCCAl+gAwIBAgIEAgAAuTANBgkqhkiG9w0BAQUFADBaMQswCQYDVQQGEwJJ
RTESMBAGA1UEChMJQmFsdGltb3JlMRMwEQYDVQQLEwpDeWJlclRydXN0MSIwIAYD
VQQDExlCYWx0aW1vcmUgQ3liZXJUcnVzdCBSb290MB4XDTAwMDUxMjE4NDYwMFoX
DTI1MDUxMjIzNTkwMFowWjELMAkGA1UEBhMCSUUxEjAQBgNVBAoTCUJhbHRpbW9y
ZTETMBEGA1UECxMKQ3liZXJUcnVzdDEiMCAGA1UEAxMZQmFsdGltb3JlIEN5YmVy
VHJ1c3QgUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKMEuyKr
mD1X6CZymrV51Cni4eiVgLGw41uOKymaZN+hXe2wCQVt2yguzmKiYv60iNoS6zjr
IZ3AQSsBUnuId9Mcj8e6uYi1agnnc+gRQKfRzMpijS3ljwumUNKoUMMo6vWrJYeK
mpYcqWe4PwzV9/lSEy/CG9VwcPCPwBLKBsua4dnKM3p31vjsufFoREJIE9LAwqSu
XmD+tqYF/LTdB1kC1FkYmGP1pWPgkAx9XbIGevOF6uvUA65ehD5f/xXtabz5OTZy
dc93Uk3zyZAsuT3lySNTPx8kmCFcB5kpvcY67Oduhjprl3RjM71oGDHweI12v/ye
jl0qhqdNkNwnGjkCAwEAAaNFMEMwHQYDVR0OBBYEFOWdWTCCR1jMrPoIVDaGezq1
BE3wMBIGA1UdEwEB/wQIMAYBAf8CAQMwDgYDVR0PAQH/BAQDAgEGMA0GCSqGSIb3
DQEBBQUAA4IBAQCFDF2O5G9RaEIFoN27TyclhAO992T9Ldcw46QQF+vaKSm2eT92
9hkTI7gQCvlYpNRhcL0EYWoSihfVCr3FvDB81ukMJY2GQE/szKN+OMY3EU/t3Wgx
jkzSswF07r51XgdIGn9w/xZchMB5hbgF/X++ZRGjD8ACtPhSNzkE1akxehi/oCr0
Epn3o0WC4zxe9Z2etciefC7IpJ5OCBRLbf1wbWsaY71k5h+3zvDyny67G7fyUIhz
ksLi4xaNmjICq44Y3ekQEe5+NauQrz4wlHrQMz2nZQ/1/I6eYs9HRCwBXbsdtTLS
R9I4LtD+gdwyah617jzV/OeBHRnDJELqYzmp
-----END CERTIFICATE-----

3
application/src/main/java/org/thingsboard/server/ThingsboardInstallApplication.java

@ -29,7 +29,8 @@ import java.util.Arrays;
@ComponentScan({"org.thingsboard.server.install",
"org.thingsboard.server.service.component",
"org.thingsboard.server.service.install",
"org.thingsboard.server.dao"})
"org.thingsboard.server.dao",
"org.thingsboard.server.common.stats"})
public class ThingsboardInstallApplication {
private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name";

15
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -221,6 +221,10 @@ public class ActorSystemContext {
@Getter
private ClaimDevicesService claimDevicesService;
@Autowired
@Getter
private JsInvokeStats jsInvokeStats;
//TODO: separate context for TbCore and TbRuleEngine
@Autowired(required = false)
@Getter
@ -284,19 +288,14 @@ public class ActorSystemContext {
@Getter
private long statisticsPersistFrequency;
@Getter
private final AtomicInteger jsInvokeRequestsCount = new AtomicInteger(0);
@Getter
private final AtomicInteger jsInvokeResponsesCount = new AtomicInteger(0);
@Getter
private final AtomicInteger jsInvokeFailuresCount = new AtomicInteger(0);
@Scheduled(fixedDelayString = "${actors.statistics.js_print_interval_ms}")
public void printStats() {
if (statisticsEnabled) {
if (jsInvokeRequestsCount.get() > 0 || jsInvokeResponsesCount.get() > 0 || jsInvokeFailuresCount.get() > 0) {
if (jsInvokeStats.getRequests() > 0 || jsInvokeStats.getResponses() > 0 || jsInvokeStats.getFailures() > 0) {
log.info("Rule Engine JS Invoke Stats: requests [{}] responses [{}] failures [{}]",
jsInvokeRequestsCount.getAndSet(0), jsInvokeResponsesCount.getAndSet(0), jsInvokeFailuresCount.getAndSet(0));
jsInvokeStats.getRequests(), jsInvokeStats.getResponses(), jsInvokeStats.getFailures());
jsInvokeStats.reset();
}
}
}

17
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java

@ -132,6 +132,9 @@ class DefaultTbContext implements TbContext {
.setTenantIdMSB(getTenantId().getId().getMostSignificantBits())
.setTenantIdLSB(getTenantId().getId().getLeastSignificantBits())
.setTbMsg(TbMsg.toByteString(tbMsg)).build();
if (nodeCtx.getSelf().isDebugMode()) {
mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), tbMsg, "To Root Rule Chain");
}
mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, new SimpleTbQueueCallback(onSuccess, onFailure));
}
@ -177,10 +180,10 @@ class DefaultTbContext implements TbContext {
enqueueForTellNext(tpi, tbMsg, relationTypes, null, onSuccess, onFailure);
}
private void enqueueForTellNext(TopicPartitionInfo tpi, TbMsg tbMsg, Set<String> relationTypes, String failureMessage, Runnable onSuccess, Consumer<Throwable> onFailure) {
private void enqueueForTellNext(TopicPartitionInfo tpi, TbMsg source, Set<String> relationTypes, String failureMessage, Runnable onSuccess, Consumer<Throwable> onFailure) {
RuleChainId ruleChainId = nodeCtx.getSelf().getRuleChainId();
RuleNodeId ruleNodeId = nodeCtx.getSelf().getId();
tbMsg = TbMsg.newMsg(tbMsg, ruleChainId, ruleNodeId);
TbMsg tbMsg = TbMsg.newMsg(source, ruleChainId, ruleNodeId);
TransportProtos.ToRuleEngineMsg.Builder msg = TransportProtos.ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(getTenantId().getId().getMostSignificantBits())
.setTenantIdLSB(getTenantId().getId().getLeastSignificantBits())
@ -189,6 +192,10 @@ class DefaultTbContext implements TbContext {
if (failureMessage != null) {
msg.setFailureMessage(failureMessage);
}
if (nodeCtx.getSelf().isDebugMode()) {
relationTypes.forEach(relationType ->
mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), tbMsg, relationType));
}
mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg.build(), new SimpleTbQueueCallback(onSuccess, onFailure));
}
@ -294,21 +301,21 @@ class DefaultTbContext implements TbContext {
@Override
public void logJsEvalRequest() {
if (mainCtx.isStatisticsEnabled()) {
mainCtx.getJsInvokeRequestsCount().incrementAndGet();
mainCtx.getJsInvokeStats().incrementRequests();
}
}
@Override
public void logJsEvalResponse() {
if (mainCtx.isStatisticsEnabled()) {
mainCtx.getJsInvokeResponsesCount().incrementAndGet();
mainCtx.getJsInvokeStats().incrementResponses();
}
}
@Override
public void logJsEvalFailure() {
if (mainCtx.isStatisticsEnabled()) {
mainCtx.getJsInvokeFailuresCount().incrementAndGet();
mainCtx.getJsInvokeStats().incrementFailures();
}
}

2
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java

@ -168,7 +168,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
private TbActorRef createRuleNodeActor(TbActorCtx ctx, RuleNode ruleNode) {
return ctx.getOrCreateChildActor(new TbEntityActorId(ruleNode.getId()),
() -> DefaultActorService.RULE_DISPATCHER_NAME,
() -> new RuleNodeActor.ActorCreator(systemContext, tenantId, entityId, ruleNode.getName(), ruleNode.getId()));
() -> new RuleNodeActor.ActorCreator(systemContext, tenantId, entityId, ruleChainName, ruleNode.getId()));
}
private void initRoutes(RuleChain ruleChain, List<RuleNode> ruleNodeList) {

5
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java

@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.plugin.ComponentLifecycleState;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.common.msg.queue.PartitionChangeMsg;
import org.thingsboard.server.common.msg.queue.RuleNodeException;
import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
/**
* @author Andrew Shvayka
@ -38,6 +39,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
private RuleNode ruleNode;
private TbNode tbNode;
private DefaultTbContext defaultCtx;
private RuleNodeInfo info;
RuleNodeActorMessageProcessor(TenantId tenantId, String ruleChainName, RuleNodeId ruleNodeId, ActorSystemContext systemContext
, TbActorRef parent, TbActorRef self) {
@ -46,6 +48,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
this.self = self;
this.ruleNode = systemContext.getRuleChainService().findRuleNodeById(tenantId, entityId);
this.defaultCtx = new DefaultTbContext(systemContext, new RuleNodeCtx(tenantId, parent, self, ruleNode));
this.info = new RuleNodeInfo(ruleNodeId, ruleChainName, ruleNode != null ? ruleNode.getName() : "Unknown");
}
@Override
@ -59,6 +62,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
@Override
public void onUpdate(TbActorCtx context) throws Exception {
RuleNode newRuleNode = systemContext.getRuleChainService().findRuleNodeById(tenantId, entityId);
this.info = new RuleNodeInfo(entityId, ruleChainName, newRuleNode != null ? newRuleNode.getName() : "Unknown");
boolean restartRequired = state != ComponentLifecycleState.ACTIVE ||
!(ruleNode.getType().equals(newRuleNode.getType()) && ruleNode.getConfiguration().equals(newRuleNode.getConfiguration()));
this.ruleNode = newRuleNode;
@ -99,6 +103,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
}
void onRuleChainToRuleNodeMsg(RuleChainToRuleNodeMsg msg) throws Exception {
msg.getMsg().getCallback().visit(info);
checkActive(msg.getMsg());
if (ruleNode.isDebugMode()) {
systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), msg.getFromRelationType());

19
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -26,21 +26,12 @@ import org.thingsboard.server.common.msg.MsgType;
import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto;
import org.thingsboard.server.gen.transport.TransportProtos.EdgeNotificationMsgProto;
import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto;
import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto;
import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg;
import org.thingsboard.server.gen.transport.TransportProtos.*;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.edge.EdgeNotificationService;
import org.thingsboard.server.service.encoding.DataDecodingEncodingService;
@ -83,19 +74,20 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
private final TbLocalSubscriptionService localSubscriptionService;
private final SubscriptionManagerService subscriptionManagerService;
private final TbCoreDeviceRpcService tbCoreDeviceRpcService;
private final TbCoreConsumerStats stats;
private final EdgeNotificationService edgeNotificationService;
private final TbCoreConsumerStats stats = new TbCoreConsumerStats();
public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext,
DeviceStateService stateService, TbLocalSubscriptionService localSubscriptionService,
SubscriptionManagerService subscriptionManagerService, DataDecodingEncodingService encodingService,
TbCoreDeviceRpcService tbCoreDeviceRpcService, EdgeNotificationService edgeNotificationService) {
TbCoreDeviceRpcService tbCoreDeviceRpcService, StatsFactory statsFactory, EdgeNotificationService edgeNotificationService) {
super(actorContext, encodingService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer());
this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer();
this.stateService = stateService;
this.localSubscriptionService = localSubscriptionService;
this.subscriptionManagerService = subscriptionManagerService;
this.tbCoreDeviceRpcService = tbCoreDeviceRpcService;
this.stats = new TbCoreConsumerStats(statsFactory);
this.edgeNotificationService = edgeNotificationService;
}
@ -235,6 +227,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
public void printStats() {
if (statsEnabled) {
stats.printStats();
stats.reset();
}
}

34
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@ -22,11 +22,13 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.thingsboard.rule.engine.api.RpcError;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg;
import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
import org.thingsboard.server.common.msg.queue.ServiceQueue;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
@ -40,6 +42,7 @@ import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.queue.util.TbRuleEngineComponent;
import org.thingsboard.server.service.encoding.DataDecodingEncodingService;
import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
@ -58,6 +61,7 @@ import javax.annotation.PreDestroy;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
@ -79,6 +83,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
@Value("${queue.rule-engine.stats.enabled:true}")
private boolean statsEnabled;
private final StatsFactory statsFactory;
private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory;
private final TbRuleEngineProcessingStrategyFactory processingStrategyFactory;
private final TbRuleEngineQueueFactory tbRuleEngineQueueFactory;
@ -95,7 +100,8 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
TbQueueRuleEngineSettings ruleEngineSettings,
TbRuleEngineQueueFactory tbRuleEngineQueueFactory, RuleEngineStatisticsService statisticsService,
ActorSystemContext actorContext, DataDecodingEncodingService encodingService,
TbRuleEngineDeviceRpcService tbDeviceRpcService) {
TbRuleEngineDeviceRpcService tbDeviceRpcService,
StatsFactory statsFactory) {
super(actorContext, encodingService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer());
this.statisticsService = statisticsService;
this.ruleEngineSettings = ruleEngineSettings;
@ -103,6 +109,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
this.submitStrategyFactory = submitStrategyFactory;
this.processingStrategyFactory = processingStrategyFactory;
this.tbDeviceRpcService = tbDeviceRpcService;
this.statsFactory = statsFactory;
}
@PostConstruct
@ -111,7 +118,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
for (TbRuleEngineQueueConfiguration configuration : ruleEngineSettings.getQueues()) {
consumerConfigurations.putIfAbsent(configuration.getName(), configuration);
consumers.computeIfAbsent(configuration.getName(), queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration));
consumerStats.put(configuration.getName(), new TbRuleEngineConsumerStats(configuration.getName()));
consumerStats.put(configuration.getName(), new TbRuleEngineConsumerStats(configuration.getName(), statsFactory));
}
submitExecutor = Executors.newSingleThreadExecutor();
}
@ -181,6 +188,12 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
}
TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(configuration.getName(), timeout, ctx);
if (timeout) {
printFirstOrAll(configuration, ctx, ctx.getPendingMap(), "Timeout");
}
if (!ctx.getFailedMap().isEmpty()) {
printFirstOrAll(configuration, ctx, ctx.getFailedMap(), "Failed");
}
TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result);
if (statsEnabled) {
stats.log(result, decision.isCommit());
@ -208,6 +221,22 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
});
}
private void printFirstOrAll(TbRuleEngineQueueConfiguration configuration, TbMsgPackProcessingContext ctx, Map<UUID, TbProtoQueueMsg<ToRuleEngineMsg>> map, String prefix) {
boolean printAll = log.isTraceEnabled();
log.info("{} to process [{}] messages", prefix, map.size());
for (Map.Entry<UUID, TbProtoQueueMsg<ToRuleEngineMsg>> pending : map.entrySet()) {
ToRuleEngineMsg tmp = pending.getValue().getValue();
TbMsg tmpMsg = TbMsg.fromBytes(configuration.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY);
RuleNodeInfo ruleNodeInfo = ctx.getLastVisitedRuleNode(pending.getKey());
if (printAll) {
log.trace("[{}] {} to process message: {}, Last Rule Node: {}", new TenantId(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo);
} else {
log.info("[{}] {} to process message: {}, Last Rule Node: {}", new TenantId(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo);
break;
}
}
}
@Override
protected ServiceType getServiceType() {
return ServiceType.TB_RULE_ENGINE;
@ -269,6 +298,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
consumerStats.forEach((queue, stats) -> {
stats.printStats();
statisticsService.reportQueueStats(ts, stats);
stats.reset();
});
}
}

125
application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java

@ -17,83 +17,132 @@ package org.thingsboard.server.service.queue;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.common.stats.StatsCounter;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.stats.StatsType;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.*;
@Slf4j
public class TbCoreConsumerStats {
public static final String TOTAL_MSGS = "totalMsgs";
public static final String SESSION_EVENTS = "sessionEvents";
public static final String GET_ATTRIBUTE = "getAttr";
public static final String ATTRIBUTE_SUBSCRIBES = "subToAttr";
public static final String RPC_SUBSCRIBES = "subToRpc";
public static final String TO_DEVICE_RPC_CALL_RESPONSES = "toDevRpc";
public static final String SUBSCRIPTION_INFO = "subInfo";
public static final String DEVICE_CLAIMS = "claimDevice";
public static final String DEVICE_STATES = "deviceState";
public static final String SUBSCRIPTION_MSGS = "subMsgs";
public static final String TO_CORE_NOTIFICATIONS = "coreNfs";
public static final String EDGE_NOTIFICATIONS = "edgeNfs";
private final AtomicInteger totalCounter = new AtomicInteger(0);
private final AtomicInteger sessionEventCounter = new AtomicInteger(0);
private final AtomicInteger getAttributesCounter = new AtomicInteger(0);
private final AtomicInteger subscribeToAttributesCounter = new AtomicInteger(0);
private final AtomicInteger subscribeToRPCCounter = new AtomicInteger(0);
private final AtomicInteger toDeviceRPCCallResponseCounter = new AtomicInteger(0);
private final AtomicInteger subscriptionInfoCounter = new AtomicInteger(0);
private final AtomicInteger claimDeviceCounter = new AtomicInteger(0);
private final StatsCounter totalCounter;
private final StatsCounter sessionEventCounter;
private final StatsCounter getAttributesCounter;
private final StatsCounter subscribeToAttributesCounter;
private final StatsCounter subscribeToRPCCounter;
private final StatsCounter toDeviceRPCCallResponseCounter;
private final StatsCounter subscriptionInfoCounter;
private final StatsCounter claimDeviceCounter;
private final AtomicInteger deviceStateCounter = new AtomicInteger(0);
private final AtomicInteger subscriptionMsgCounter = new AtomicInteger(0);
private final AtomicInteger toCoreNotificationsCounter = new AtomicInteger(0);
private final AtomicInteger edgeNotificationMsgCounter = new AtomicInteger(0);
private final StatsCounter deviceStateCounter;
private final StatsCounter subscriptionMsgCounter;
private final StatsCounter toCoreNotificationsCounter;
private final StatsCounter edgeNotificationMsgCounter;
private final List<StatsCounter> counters = new ArrayList<>();
public TbCoreConsumerStats(StatsFactory statsFactory) {
String statsKey = StatsType.CORE.getName();
this.totalCounter = statsFactory.createStatsCounter(statsKey, TOTAL_MSGS);
this.sessionEventCounter = statsFactory.createStatsCounter(statsKey, SESSION_EVENTS);
this.getAttributesCounter = statsFactory.createStatsCounter(statsKey, GET_ATTRIBUTE);
this.subscribeToAttributesCounter = statsFactory.createStatsCounter(statsKey, ATTRIBUTE_SUBSCRIBES);
this.subscribeToRPCCounter = statsFactory.createStatsCounter(statsKey, RPC_SUBSCRIBES);
this.toDeviceRPCCallResponseCounter = statsFactory.createStatsCounter(statsKey, TO_DEVICE_RPC_CALL_RESPONSES);
this.subscriptionInfoCounter = statsFactory.createStatsCounter(statsKey, SUBSCRIPTION_INFO);
this.claimDeviceCounter = statsFactory.createStatsCounter(statsKey, DEVICE_CLAIMS);
this.deviceStateCounter = statsFactory.createStatsCounter(statsKey, DEVICE_STATES);
this.subscriptionMsgCounter = statsFactory.createStatsCounter(statsKey, SUBSCRIPTION_MSGS);
this.toCoreNotificationsCounter = statsFactory.createStatsCounter(statsKey, TO_CORE_NOTIFICATIONS);
this.edgeNotificationMsgCounter = statsFactory.createStatsCounter(statsKey, EDGE_NOTIFICATIONS);
counters.add(totalCounter);
counters.add(sessionEventCounter);
counters.add(getAttributesCounter);
counters.add(subscribeToAttributesCounter);
counters.add(subscribeToRPCCounter);
counters.add(toDeviceRPCCallResponseCounter);
counters.add(subscriptionInfoCounter);
counters.add(claimDeviceCounter);
counters.add(deviceStateCounter);
counters.add(subscriptionMsgCounter);
counters.add(toCoreNotificationsCounter);
counters.add(edgeNotificationMsgCounter);
}
public void log(TransportProtos.TransportToDeviceActorMsg msg) {
totalCounter.incrementAndGet();
totalCounter.increment();
if (msg.hasSessionEvent()) {
sessionEventCounter.incrementAndGet();
sessionEventCounter.increment();
}
if (msg.hasGetAttributes()) {
getAttributesCounter.incrementAndGet();
getAttributesCounter.increment();
}
if (msg.hasSubscribeToAttributes()) {
subscribeToAttributesCounter.incrementAndGet();
subscribeToAttributesCounter.increment();
}
if (msg.hasSubscribeToRPC()) {
subscribeToRPCCounter.incrementAndGet();
subscribeToRPCCounter.increment();
}
if (msg.hasToDeviceRPCCallResponse()) {
toDeviceRPCCallResponseCounter.incrementAndGet();
toDeviceRPCCallResponseCounter.increment();
}
if (msg.hasSubscriptionInfo()) {
subscriptionInfoCounter.incrementAndGet();
subscriptionInfoCounter.increment();
}
if (msg.hasClaimDevice()) {
claimDeviceCounter.incrementAndGet();
claimDeviceCounter.increment();
}
}
public void log(TransportProtos.DeviceStateServiceMsgProto msg) {
totalCounter.incrementAndGet();
deviceStateCounter.incrementAndGet();
totalCounter.increment();
deviceStateCounter.increment();
}
public void log(TransportProtos.EdgeNotificationMsgProto msg) {
totalCounter.incrementAndGet();
edgeNotificationMsgCounter.incrementAndGet();
totalCounter.increment();
edgeNotificationMsgCounter.increment();
}
public void log(TransportProtos.SubscriptionMgrMsgProto msg) {
totalCounter.incrementAndGet();
subscriptionMsgCounter.incrementAndGet();
totalCounter.increment();
subscriptionMsgCounter.increment();
}
public void log(TransportProtos.ToCoreNotificationMsg msg) {
totalCounter.incrementAndGet();
toCoreNotificationsCounter.incrementAndGet();
totalCounter.increment();
toCoreNotificationsCounter.increment();
}
public void printStats() {
int total = totalCounter.getAndSet(0);
int total = totalCounter.get();
if (total > 0) {
log.info("Total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]" +
" deviceState [{}] subMgr [{}] coreNfs [{}] edgeNfs [{}]",
total, sessionEventCounter.getAndSet(0),
getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0),
subscribeToRPCCounter.getAndSet(0), toDeviceRPCCallResponseCounter.getAndSet(0),
subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0)
, deviceStateCounter.getAndSet(0), subscriptionMsgCounter.getAndSet(0), toCoreNotificationsCounter.getAndSet(0),
edgeNotificationMsgCounter.getAndSet(0));
StringBuilder stats = new StringBuilder();
counters.forEach(counter -> {
stats.append(counter.getName()).append(" = [").append(counter.get()).append("] ");
});
log.info("Core Stats: {}", stats);
}
}
public void reset() {
counters.forEach(StatsCounter::clear);
}
}

8
application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java

@ -16,8 +16,10 @@
package org.thingsboard.server.service.queue;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import java.util.UUID;
@ -45,4 +47,10 @@ public class TbMsgPackCallback implements TbMsgCallback {
log.trace("[{}] ON FAILURE", id, e);
ctx.onFailure(tenantId, id, e);
}
@Override
public void visit(RuleNodeInfo ruleNodeInfo) {
log.trace("[{}] ON PROCESS: {}", id, ruleNodeInfo);
ctx.visit(id, ruleNodeInfo);
}
}

14
application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java

@ -16,8 +16,10 @@
package org.thingsboard.server.service.queue;
import lombok.Getter;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy;
@ -32,7 +34,6 @@ import java.util.concurrent.atomic.AtomicInteger;
public class TbMsgPackProcessingContext {
private final TbRuleEngineSubmitStrategy submitStrategy;
private final AtomicInteger pendingCount;
private final CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
@Getter
@ -44,6 +45,8 @@ public class TbMsgPackProcessingContext {
@Getter
private final ConcurrentMap<TenantId, RuleEngineException> exceptionsMap = new ConcurrentHashMap<>();
private final ConcurrentMap<UUID, RuleNodeInfo> lastRuleNodeMap = new ConcurrentHashMap<>();
public TbMsgPackProcessingContext(TbRuleEngineSubmitStrategy submitStrategy) {
this.submitStrategy = submitStrategy;
this.pendingMap = submitStrategy.getPendingMap();
@ -81,4 +84,13 @@ public class TbMsgPackProcessingContext {
processingTimeoutLatch.countDown();
}
}
public void visit(UUID id, RuleNodeInfo ruleNodeInfo) {
lastRuleNodeMap.put(id, ruleNodeInfo);
}
public RuleNodeInfo getLastVisitedRuleNode(UUID id) {
return lastRuleNodeMap.get(id);
}
}

95
application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java

@ -15,23 +15,21 @@
*/
package org.thingsboard.server.service.queue;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult;
import org.thingsboard.server.common.stats.StatsCounter;
import org.thingsboard.server.common.stats.StatsType;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
@Slf4j
@Data
public class TbRuleEngineConsumerStats {
public static final String TOTAL_MSGS = "totalMsgs";
@ -43,61 +41,72 @@ public class TbRuleEngineConsumerStats {
public static final String SUCCESSFUL_ITERATIONS = "successfulIterations";
public static final String FAILED_ITERATIONS = "failedIterations";
private final AtomicInteger totalMsgCounter = new AtomicInteger(0);
private final AtomicInteger successMsgCounter = new AtomicInteger(0);
private final AtomicInteger tmpTimeoutMsgCounter = new AtomicInteger(0);
private final AtomicInteger tmpFailedMsgCounter = new AtomicInteger(0);
private final StatsCounter totalMsgCounter;
private final StatsCounter successMsgCounter;
private final StatsCounter tmpTimeoutMsgCounter;
private final StatsCounter tmpFailedMsgCounter;
private final AtomicInteger timeoutMsgCounter = new AtomicInteger(0);
private final AtomicInteger failedMsgCounter = new AtomicInteger(0);
private final StatsCounter timeoutMsgCounter;
private final StatsCounter failedMsgCounter;
private final AtomicInteger successIterationsCounter = new AtomicInteger(0);
private final AtomicInteger failedIterationsCounter = new AtomicInteger(0);
private final StatsCounter successIterationsCounter;
private final StatsCounter failedIterationsCounter;
private final Map<String, AtomicInteger> counters = new HashMap<>();
private final List<StatsCounter> counters = new ArrayList<>();
private final ConcurrentMap<UUID, TbTenantRuleEngineStats> tenantStats = new ConcurrentHashMap<>();
private final ConcurrentMap<TenantId, RuleEngineException> tenantExceptions = new ConcurrentHashMap<>();
private final String queueName;
public TbRuleEngineConsumerStats(String queueName) {
public TbRuleEngineConsumerStats(String queueName, StatsFactory statsFactory) {
this.queueName = queueName;
counters.put(TOTAL_MSGS, totalMsgCounter);
counters.put(SUCCESSFUL_MSGS, successMsgCounter);
counters.put(TIMEOUT_MSGS, timeoutMsgCounter);
counters.put(FAILED_MSGS, failedMsgCounter);
counters.put(TMP_TIMEOUT, tmpTimeoutMsgCounter);
counters.put(TMP_FAILED, tmpFailedMsgCounter);
counters.put(SUCCESSFUL_ITERATIONS, successIterationsCounter);
counters.put(FAILED_ITERATIONS, failedIterationsCounter);
String statsKey = StatsType.RULE_ENGINE.getName() + "." + queueName;
this.totalMsgCounter = statsFactory.createStatsCounter(statsKey, TOTAL_MSGS);
this.successMsgCounter = statsFactory.createStatsCounter(statsKey, SUCCESSFUL_MSGS);
this.timeoutMsgCounter = statsFactory.createStatsCounter(statsKey, TIMEOUT_MSGS);
this.failedMsgCounter = statsFactory.createStatsCounter(statsKey, FAILED_MSGS);
this.tmpTimeoutMsgCounter = statsFactory.createStatsCounter(statsKey, TMP_TIMEOUT);
this.tmpFailedMsgCounter = statsFactory.createStatsCounter(statsKey, TMP_FAILED);
this.successIterationsCounter = statsFactory.createStatsCounter(statsKey, SUCCESSFUL_ITERATIONS);
this.failedIterationsCounter = statsFactory.createStatsCounter(statsKey, FAILED_ITERATIONS);
counters.add(totalMsgCounter);
counters.add(successMsgCounter);
counters.add(timeoutMsgCounter);
counters.add(failedMsgCounter);
counters.add(tmpTimeoutMsgCounter);
counters.add(tmpFailedMsgCounter);
counters.add(successIterationsCounter);
counters.add(failedIterationsCounter);
}
public void log(TbRuleEngineProcessingResult msg, boolean finalIterationForPack) {
int success = msg.getSuccessMap().size();
int pending = msg.getPendingMap().size();
int failed = msg.getFailedMap().size();
totalMsgCounter.addAndGet(success + pending + failed);
successMsgCounter.addAndGet(success);
totalMsgCounter.add(success + pending + failed);
successMsgCounter.add(success);
msg.getSuccessMap().values().forEach(m -> getTenantStats(m).logSuccess());
if (finalIterationForPack) {
if (pending > 0 || failed > 0) {
timeoutMsgCounter.addAndGet(pending);
failedMsgCounter.addAndGet(failed);
timeoutMsgCounter.add(pending);
failedMsgCounter.add(failed);
if (pending > 0) {
msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTimeout());
}
if (failed > 0) {
msg.getFailedMap().values().forEach(m -> getTenantStats(m).logFailed());
}
failedIterationsCounter.incrementAndGet();
failedIterationsCounter.increment();
} else {
successIterationsCounter.incrementAndGet();
successIterationsCounter.increment();
}
} else {
failedIterationsCounter.incrementAndGet();
tmpTimeoutMsgCounter.addAndGet(pending);
tmpFailedMsgCounter.addAndGet(failed);
failedIterationsCounter.increment();
tmpTimeoutMsgCounter.add(pending);
tmpFailedMsgCounter.add(failed);
if (pending > 0) {
msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTmpTimeout());
}
@ -113,19 +122,31 @@ public class TbRuleEngineConsumerStats {
return tenantStats.computeIfAbsent(new UUID(reMsg.getTenantIdMSB(), reMsg.getTenantIdLSB()), TbTenantRuleEngineStats::new);
}
public ConcurrentMap<UUID, TbTenantRuleEngineStats> getTenantStats() {
return tenantStats;
}
public String getQueueName() {
return queueName;
}
public ConcurrentMap<TenantId, RuleEngineException> getTenantExceptions() {
return tenantExceptions;
}
public void printStats() {
int total = totalMsgCounter.get();
if (total > 0) {
StringBuilder stats = new StringBuilder();
counters.forEach((label, value) -> {
stats.append(label).append(" = [").append(value.get()).append("] ");
counters.forEach(counter -> {
stats.append(counter.getName()).append(" = [").append(counter.get()).append("] ");
});
log.info("[{}] Stats: {}", queueName, stats);
}
}
public void reset() {
counters.values().forEach(counter -> counter.set(0));
counters.forEach(StatsCounter::clear);
tenantStats.clear();
tenantExceptions.clear();
}

4
application/src/main/java/org/thingsboard/server/service/security/model/token/RawAccessJwtToken.java

@ -52,10 +52,10 @@ public class RawAccessJwtToken implements JwtToken, Serializable {
try {
return Jwts.parser().setSigningKey(signingKey).parseClaimsJws(this.token);
} catch (UnsupportedJwtException | MalformedJwtException | IllegalArgumentException | SignatureException ex) {
log.error("Invalid JWT Token", ex);
log.debug("Invalid JWT Token", ex);
throw new BadCredentialsException("Invalid JWT token: ", ex);
} catch (ExpiredJwtException expiredEx) {
log.info("JWT Token is expired", expiredEx);
log.debug("JWT Token is expired", expiredEx);
throw new JwtExpiredTokenException(this, "JWT Token expired", expiredEx);
}
}

84
application/src/main/java/org/thingsboard/server/service/stats/DefaultJsInvokeStats.java

@ -0,0 +1,84 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.stats;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.thingsboard.server.actors.JsInvokeStats;
import org.thingsboard.server.common.stats.StatsCounter;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.stats.StatsType;
import javax.annotation.PostConstruct;
@Service
public class DefaultJsInvokeStats implements JsInvokeStats {
private static final String REQUESTS = "requests";
private static final String RESPONSES = "responses";
private static final String FAILURES = "failures";
private StatsCounter requestsCounter;
private StatsCounter responsesCounter;
private StatsCounter failuresCounter;
@Autowired
private StatsFactory statsFactory;
@PostConstruct
public void init() {
String key = StatsType.JS_INVOKE.getName();
this.requestsCounter = statsFactory.createStatsCounter(key, REQUESTS);
this.responsesCounter = statsFactory.createStatsCounter(key, RESPONSES);
this.failuresCounter = statsFactory.createStatsCounter(key, FAILURES);
}
@Override
public void incrementRequests(int amount) {
requestsCounter.add(amount);
}
@Override
public void incrementResponses(int amount) {
responsesCounter.add(amount);
}
@Override
public void incrementFailures(int amount) {
failuresCounter.add(amount);
}
@Override
public int getRequests() {
return requestsCounter.get();
}
@Override
public int getResponses() {
return responsesCounter.get();
}
@Override
public int getFailures() {
return failuresCounter.get();
}
@Override
public void reset() {
requestsCounter.clear();
responsesCounter.clear();
failuresCounter.clear();
}
}

1
application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java

@ -104,7 +104,6 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS
}
}
});
ruleEngineStats.reset();
}
private AssetId getServiceAssetId(TenantId tenantId, String queueName) {

12
application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java

@ -20,6 +20,9 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.stats.MessagesStats;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.stats.StatsType;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueResponseTemplate;
@ -41,9 +44,9 @@ import java.util.concurrent.*;
@Service
@TbCoreComponent
public class TbCoreTransportApiService {
private final TbCoreQueueFactory tbCoreQueueFactory;
private final TransportApiService transportApiService;
private final StatsFactory statsFactory;
@Value("${queue.transport_api.max_pending_requests:10000}")
private int maxPendingRequests;
@ -58,9 +61,10 @@ public class TbCoreTransportApiService {
private TbQueueResponseTemplate<TbProtoQueueMsg<TransportApiRequestMsg>,
TbProtoQueueMsg<TransportApiResponseMsg>> transportApiTemplate;
public TbCoreTransportApiService(TbCoreQueueFactory tbCoreQueueFactory, TransportApiService transportApiService) {
public TbCoreTransportApiService(TbCoreQueueFactory tbCoreQueueFactory, TransportApiService transportApiService, StatsFactory statsFactory) {
this.tbCoreQueueFactory = tbCoreQueueFactory;
this.transportApiService = transportApiService;
this.statsFactory = statsFactory;
}
@PostConstruct
@ -69,6 +73,9 @@ public class TbCoreTransportApiService {
TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> producer = tbCoreQueueFactory.createTransportApiResponseProducer();
TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> consumer = tbCoreQueueFactory.createTransportApiRequestConsumer();
String key = StatsType.TRANSPORT.getName();
MessagesStats queueStats = statsFactory.createMessagesStats(key);
DefaultTbQueueResponseTemplate.DefaultTbQueueResponseTemplateBuilder
<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> builder = DefaultTbQueueResponseTemplate.builder();
builder.requestTemplate(consumer);
@ -78,6 +85,7 @@ public class TbCoreTransportApiService {
builder.pollInterval(responsePollDuration);
builder.executor(transportCallbackExecutor);
builder.handler(transportApiService);
builder.stats(queueStats);
transportApiTemplate = builder.build();
}

1
application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java

@ -27,7 +27,6 @@ import java.sql.Statement;
@Slf4j
@PsqlDao
public abstract class AbstractCleanUpService {
@Value("${spring.datasource.url}")

2
application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java

@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.PsqlDao;
import org.thingsboard.server.dao.util.SqlDao;
import org.thingsboard.server.service.ttl.AbstractCleanUpService;
import java.sql.Connection;
@ -27,6 +28,7 @@ import java.sql.DriverManager;
import java.sql.SQLException;
@PsqlDao
@SqlDao
@Slf4j
@Service
public class EventsCleanUpService extends AbstractCleanUpService {

2
application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java

@ -18,14 +18,12 @@ package org.thingsboard.server.service.ttl.timeseries;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.thingsboard.server.dao.util.PsqlTsAnyDao;
import org.thingsboard.server.service.ttl.AbstractCleanUpService;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
@PsqlTsAnyDao
@Slf4j
public abstract class AbstractTimeseriesCleanUpService extends AbstractCleanUpService {

16
application/src/main/resources/thingsboard.yml

@ -252,14 +252,17 @@ sql:
batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}"
batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}"
stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}"
batch_threads: "${SQL_ATTRIBUTES_BATCH_THREADS:4}"
ts:
batch_size: "${SQL_TS_BATCH_SIZE:10000}"
batch_max_delay: "${SQL_TS_BATCH_MAX_DELAY_MS:100}"
stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}"
batch_threads: "${SQL_TS_BATCH_THREADS:4}"
ts_latest:
batch_size: "${SQL_TS_LATEST_BATCH_SIZE:10000}"
batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:100}"
stats_print_interval_ms: "${SQL_TS_LATEST_BATCH_STATS_PRINT_MS:10000}"
batch_threads: "${SQL_TS_LATEST_BATCH_THREADS:4}"
# Specify whether to remove null characters from strValue of attributes and timeseries before insert
remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}"
postgres:
@ -268,6 +271,7 @@ sql:
timescale:
# Specify Interval size for new data chunks storage.
chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}"
batch_threads: "${SQL_TIMESCALE_BATCH_THREADS:4}"
ttl:
ts:
enabled: "${SQL_TTL_TS_ENABLED:true}"
@ -767,7 +771,7 @@ queue:
transport:
# For high priority notifications that require minimum latency and processing time
notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}"
poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}"
poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}"
service:
type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine
@ -775,3 +779,13 @@ service:
id: "${TB_SERVICE_ID:}"
tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id.
metrics:
# Enable/disable actuator metrics.
enabled: "${METRICS_ENABLED:false}"
management:
endpoints:
web:
exposure:
# Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics).
include: '${METRICS_ENDPOINTS_EXPOSE:info}'

44
common/actor/src/main/java/org/thingsboard/server/actors/JsInvokeStats.java

@ -0,0 +1,44 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.actors;
public interface JsInvokeStats {
default void incrementRequests() {
incrementRequests(1);
}
void incrementRequests(int amount);
default void incrementResponses() {
incrementResponses(1);
}
void incrementResponses(int amount);
default void incrementFailures() {
incrementFailures(1);
}
void incrementFailures(int amount);
int getRequests();
int getResponses();
int getFailures();
void reset();
}

4
common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsDao.java

@ -17,5 +17,9 @@ package org.thingsboard.server.dao.util;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@Retention(RetentionPolicy.RUNTIME)
@ConditionalOnExpression("'${database.ts.type}'=='sql' && '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'")
public @interface PsqlTsDao { }

1
common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java

@ -106,7 +106,6 @@ public final class TbMsg implements Serializable {
if (callback != null) {
this.callback = callback;
} else {
log.warn("[{}] Created message with empty callback: {}", originator, type);
this.callback = TbMsgCallback.EMPTY;
}
}

31
common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeInfo.java

@ -0,0 +1,31 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.msg.queue;
import org.thingsboard.server.common.data.id.RuleNodeId;
public class RuleNodeInfo {
private final String label;
public RuleNodeInfo(RuleNodeId id, String ruleChainName, String ruleNodeName) {
this.label = "[RuleChain: " + ruleChainName + "|RuleNode: " + ruleNodeName + "(" + id + ")]";
}
@Override
public String toString() {
return label;
}
}

3
common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java

@ -34,4 +34,7 @@ public interface TbMsgCallback {
void onFailure(RuleEngineException e);
default void visit(RuleNodeInfo ruleNodeInfo) {
}
}

1
common/pom.xml

@ -41,6 +41,7 @@
<module>queue</module>
<module>transport</module>
<module>dao-api</module>
<module>stats</module>
<module>edge-api</module>
</modules>

5
common/queue/pom.xml

@ -48,6 +48,10 @@
<groupId>org.thingsboard.common</groupId>
<artifactId>message</artifactId>
</dependency>
<dependency>
<groupId>org.thingsboard.common</groupId>
<artifactId>stats</artifactId>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
@ -112,6 +116,7 @@
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>

2
common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java

@ -16,6 +16,7 @@
package org.thingsboard.server.queue;
import com.google.common.util.concurrent.ListenableFuture;
import org.thingsboard.server.common.stats.MessagesStats;
public interface TbQueueRequestTemplate<Request extends TbQueueMsg, Response extends TbQueueMsg> {
@ -25,4 +26,5 @@ public interface TbQueueRequestTemplate<Request extends TbQueueMsg, Response ext
void stop();
void setMessagesStats(MessagesStats messagesStats);
}

17
common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java

@ -28,6 +28,7 @@ import org.thingsboard.server.queue.TbQueueMsg;
import org.thingsboard.server.queue.TbQueueMsgMetadata;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.common.stats.MessagesStats;
import java.util.List;
import java.util.UUID;
@ -54,6 +55,8 @@ public class DefaultTbQueueRequestTemplate<Request extends TbQueueMsg, Response
private volatile long tickSize = 0L;
private volatile boolean stopped = false;
private MessagesStats messagesStats;
@Builder
public DefaultTbQueueRequestTemplate(TbQueueAdmin queueAdmin,
TbQueueProducer<Request> requestTemplate,
@ -153,6 +156,11 @@ public class DefaultTbQueueRequestTemplate<Request extends TbQueueMsg, Response
}
}
@Override
public void setMessagesStats(MessagesStats messagesStats) {
this.messagesStats = messagesStats;
}
@Override
public ListenableFuture<Response> send(Request request) {
if (tickSize > maxPendingRequests) {
@ -166,14 +174,23 @@ public class DefaultTbQueueRequestTemplate<Request extends TbQueueMsg, Response
ResponseMetaData<Response> responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future);
pendingRequests.putIfAbsent(requestId, responseMetaData);
log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, request.getKey(), responseMetaData.expTime);
if (messagesStats != null) {
messagesStats.incrementTotal();
}
requestTemplate.send(TopicPartitionInfo.builder().topic(requestTemplate.getDefaultTopic()).build(), request, new TbQueueCallback() {
@Override
public void onSuccess(TbQueueMsgMetadata metadata) {
if (messagesStats != null) {
messagesStats.incrementSuccessful();
}
log.trace("[{}] Request sent: {}", requestId, metadata);
}
@Override
public void onFailure(Throwable t) {
if (messagesStats != null) {
messagesStats.incrementFailed();
}
pendingRequests.remove(requestId);
future.setException(t);
}

10
common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java

@ -23,6 +23,7 @@ import org.thingsboard.server.queue.TbQueueHandler;
import org.thingsboard.server.queue.TbQueueMsg;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueResponseTemplate;
import org.thingsboard.server.common.stats.MessagesStats;
import java.util.List;
import java.util.UUID;
@ -44,6 +45,7 @@ public class DefaultTbQueueResponseTemplate<Request extends TbQueueMsg, Response
private final ExecutorService loopExecutor;
private final ScheduledExecutorService timeoutExecutor;
private final ExecutorService callbackExecutor;
private final MessagesStats stats;
private final int maxPendingRequests;
private final long requestTimeout;
@ -58,7 +60,8 @@ public class DefaultTbQueueResponseTemplate<Request extends TbQueueMsg, Response
long pollInterval,
long requestTimeout,
int maxPendingRequests,
ExecutorService executor) {
ExecutorService executor,
MessagesStats stats) {
this.requestTemplate = requestTemplate;
this.responseTemplate = responseTemplate;
this.pendingRequests = new ConcurrentHashMap<>();
@ -66,6 +69,7 @@ public class DefaultTbQueueResponseTemplate<Request extends TbQueueMsg, Response
this.pollInterval = pollInterval;
this.requestTimeout = requestTimeout;
this.callbackExecutor = executor;
this.stats = stats;
this.timeoutExecutor = Executors.newSingleThreadScheduledExecutor();
this.loopExecutor = Executors.newSingleThreadExecutor();
}
@ -108,11 +112,13 @@ public class DefaultTbQueueResponseTemplate<Request extends TbQueueMsg, Response
String responseTopic = bytesToString(responseTopicHeader);
try {
pendingRequestCount.getAndIncrement();
stats.incrementTotal();
AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(request),
response -> {
pendingRequestCount.decrementAndGet();
response.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId));
responseTemplate.send(TopicPartitionInfo.builder().topic(responseTopic).build(), response, null);
stats.incrementSuccessful();
},
e -> {
pendingRequestCount.decrementAndGet();
@ -121,6 +127,7 @@ public class DefaultTbQueueResponseTemplate<Request extends TbQueueMsg, Response
} else {
log.trace("[{}] Failed to process the request: {}", requestId, request, e);
}
stats.incrementFailed();
},
requestTimeout,
timeoutExecutor,
@ -128,6 +135,7 @@ public class DefaultTbQueueResponseTemplate<Request extends TbQueueMsg, Response
} catch (Throwable e) {
pendingRequestCount.decrementAndGet();
log.warn("[{}] Failed to process the request: {}", requestId, request, e);
stats.incrementFailed();
}
}
});

92
common/stats/pom.xml

@ -0,0 +1,92 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright © 2016-2020 The Thingsboard Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>2.5.3-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>
<artifactId>stats</artifactId>
<packaging>jar</packaging>
<name>Thingsboard Server Stats</name>
<url>https://thingsboard.io</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<main.dir>${basedir}/../..</main.dir>
</properties>
<dependencies>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
</plugins>
</build>
</project>

48
common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultCounter.java

@ -0,0 +1,48 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.stats;
import io.micrometer.core.instrument.Counter;
import java.util.concurrent.atomic.AtomicInteger;
public class DefaultCounter {
private final AtomicInteger aiCounter;
private final Counter micrometerCounter;
public DefaultCounter(AtomicInteger aiCounter, Counter micrometerCounter) {
this.aiCounter = aiCounter;
this.micrometerCounter = micrometerCounter;
}
public void increment() {
aiCounter.incrementAndGet();
micrometerCounter.increment();
}
public void clear() {
aiCounter.set(0);
}
public int get() {
return aiCounter.get();
}
public void add(int delta){
aiCounter.addAndGet(delta);
micrometerCounter.increment(delta);
}
}

65
common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultMessagesStats.java

@ -0,0 +1,65 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.stats;
public class DefaultMessagesStats implements MessagesStats {
private final StatsCounter totalCounter;
private final StatsCounter successfulCounter;
private final StatsCounter failedCounter;
public DefaultMessagesStats(StatsCounter totalCounter, StatsCounter successfulCounter, StatsCounter failedCounter) {
this.totalCounter = totalCounter;
this.successfulCounter = successfulCounter;
this.failedCounter = failedCounter;
}
@Override
public void incrementTotal(int amount) {
totalCounter.add(amount);
}
@Override
public void incrementSuccessful(int amount) {
successfulCounter.add(amount);
}
@Override
public void incrementFailed(int amount) {
failedCounter.add(amount);
}
@Override
public int getTotal() {
return totalCounter.get();
}
@Override
public int getSuccessful() {
return successfulCounter.get();
}
@Override
public int getFailed() {
return failedCounter.get();
}
@Override
public void reset() {
totalCounter.clear();
successfulCounter.clear();
failedCounter.clear();
}
}

91
common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java

@ -0,0 +1,91 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.stats;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Tags;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.util.concurrent.atomic.AtomicInteger;
@Service
public class DefaultStatsFactory implements StatsFactory {
private static final String TOTAL_MSGS = "totalMsgs";
private static final String SUCCESSFUL_MSGS = "successfulMsgs";
private static final String FAILED_MSGS = "failedMsgs";
private static final String STATS_NAME_TAG = "statsName";
private static final Counter STUB_COUNTER = new StubCounter();
@Autowired
private MeterRegistry meterRegistry;
@Value("${metrics.enabled:false}")
private Boolean metricsEnabled;
@Override
public StatsCounter createStatsCounter(String key, String statsName) {
return new StatsCounter(
new AtomicInteger(0),
metricsEnabled ?
meterRegistry.counter(key, STATS_NAME_TAG, statsName)
: STUB_COUNTER,
statsName
);
}
@Override
public DefaultCounter createDefaultCounter(String key, String... tags) {
return new DefaultCounter(
new AtomicInteger(0),
metricsEnabled ?
meterRegistry.counter(key, tags)
: STUB_COUNTER
);
}
@Override
public <T extends Number> T createGauge(String key, T number, String... tags) {
return meterRegistry.gauge(key, Tags.of(tags), number);
}
@Override
public MessagesStats createMessagesStats(String key) {
StatsCounter totalCounter = createStatsCounter(key, TOTAL_MSGS);
StatsCounter successfulCounter = createStatsCounter(key, SUCCESSFUL_MSGS);
StatsCounter failedCounter = createStatsCounter(key, FAILED_MSGS);
return new DefaultMessagesStats(totalCounter, successfulCounter, failedCounter);
}
private static class StubCounter implements Counter {
@Override
public void increment(double amount) {}
@Override
public double count() {
return 0;
}
@Override
public Id getId() {
return null;
}
}
}

44
common/stats/src/main/java/org/thingsboard/server/common/stats/MessagesStats.java

@ -0,0 +1,44 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.stats;
public interface MessagesStats {
default void incrementTotal() {
incrementTotal(1);
}
void incrementTotal(int amount);
default void incrementSuccessful() {
incrementSuccessful(1);
}
void incrementSuccessful(int amount);
default void incrementFailed() {
incrementFailed(1);
}
void incrementFailed(int amount);
int getTotal();
int getSuccessful();
int getFailed();
void reset();
}

33
common/stats/src/main/java/org/thingsboard/server/common/stats/StatsCounter.java

@ -0,0 +1,33 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.stats;
import io.micrometer.core.instrument.Counter;
import java.util.concurrent.atomic.AtomicInteger;
public class StatsCounter extends DefaultCounter {
private final String name;
public StatsCounter(AtomicInteger aiCounter, Counter micrometerCounter, String name) {
super(aiCounter, micrometerCounter);
this.name = name;
}
public String getName() {
return name;
}
}

26
common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java

@ -0,0 +1,26 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.stats;
public interface StatsFactory {
StatsCounter createStatsCounter(String key, String statsName);
DefaultCounter createDefaultCounter(String key, String... tags);
<T extends Number> T createGauge(String key, T number, String... tags);
MessagesStats createMessagesStats(String key);
}

30
common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.stats;
public enum StatsType {
RULE_ENGINE("ruleEngine"), CORE("core"), TRANSPORT("transport"), JS_INVOKE("jsInvoke"), RATE_EXECUTOR("rateExecutor");
private String name;
StatsType(String name) {
this.name = name;
}
public String getName() {
return name;
}
}

4
common/transport/transport-api/pom.xml

@ -40,6 +40,10 @@
<groupId>org.thingsboard.common</groupId>
<artifactId>queue</artifactId>
</dependency>
<dependency>
<groupId>org.thingsboard.common</groupId>
<artifactId>stats</artifactId>
</dependency>
<dependency>
<groupId>org.thingsboard.common</groupId>
<artifactId>data</artifactId>

51
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java

@ -55,6 +55,9 @@ import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
import org.thingsboard.server.queue.provider.TbTransportQueueFactory;
import org.thingsboard.server.common.stats.MessagesStats;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.stats.StatsType;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
@ -101,12 +104,17 @@ public class DefaultTransportService implements TransportService {
private final TbQueueProducerProvider producerProvider;
private final PartitionService partitionService;
private final TbServiceInfoProvider serviceInfoProvider;
private final StatsFactory statsFactory;
protected TbQueueRequestTemplate<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> transportApiRequestTemplate;
protected TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> ruleEngineMsgProducer;
protected TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> tbCoreMsgProducer;
protected TbQueueConsumer<TbProtoQueueMsg<ToTransportMsg>> transportNotificationsConsumer;
protected MessagesStats ruleEngineProducerStats;
protected MessagesStats tbCoreProducerStats;
protected MessagesStats transportApiStats;
protected ScheduledExecutorService schedulerExecutor;
protected ExecutorService transportCallbackExecutor;
@ -119,11 +127,12 @@ public class DefaultTransportService implements TransportService {
private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("transport-consumer"));
private volatile boolean stopped = false;
public DefaultTransportService(TbServiceInfoProvider serviceInfoProvider, TbTransportQueueFactory queueProvider, TbQueueProducerProvider producerProvider, PartitionService partitionService) {
public DefaultTransportService(TbServiceInfoProvider serviceInfoProvider, TbTransportQueueFactory queueProvider, TbQueueProducerProvider producerProvider, PartitionService partitionService, StatsFactory statsFactory) {
this.serviceInfoProvider = serviceInfoProvider;
this.queueProvider = queueProvider;
this.producerProvider = producerProvider;
this.partitionService = partitionService;
this.statsFactory = statsFactory;
}
@PostConstruct
@ -133,10 +142,14 @@ public class DefaultTransportService implements TransportService {
new TbRateLimits(perTenantLimitsConf);
new TbRateLimits(perDevicesLimitsConf);
}
this.ruleEngineProducerStats = statsFactory.createMessagesStats(StatsType.RULE_ENGINE.getName() + ".producer");
this.tbCoreProducerStats = statsFactory.createMessagesStats(StatsType.CORE.getName() + ".producer");
this.transportApiStats = statsFactory.createMessagesStats(StatsType.TRANSPORT.getName() + ".producer");
this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler"));
this.transportCallbackExecutor = Executors.newWorkStealingPool(20);
this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS);
transportApiRequestTemplate = queueProvider.createTransportApiRequestTemplate();
transportApiRequestTemplate.setMessagesStats(transportApiStats);
ruleEngineMsgProducer = producerProvider.getRuleEngineMsgProducer();
tbCoreMsgProducer = producerProvider.getTbCoreMsgProducer();
transportNotificationsConsumer = queueProvider.createTransportNotificationsConsumer();
@ -557,10 +570,14 @@ public class DefaultTransportService implements TransportService {
if (log.isTraceEnabled()) {
log.trace("[{}][{}] Pushing to topic {} message {}", getTenantId(sessionInfo), getDeviceId(sessionInfo), tpi.getFullTopicName(), toDeviceActorMsg);
}
TransportTbQueueCallback transportTbQueueCallback = callback != null ?
new TransportTbQueueCallback(callback) : null;
tbCoreProducerStats.incrementTotal();
StatsCallback wrappedCallback = new StatsCallback(transportTbQueueCallback, tbCoreProducerStats);
tbCoreMsgProducer.send(tpi,
new TbProtoQueueMsg<>(getRoutingKey(sessionInfo),
ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ?
new TransportTbQueueCallback(callback) : null);
ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()),
wrappedCallback);
}
protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) {
@ -571,7 +588,9 @@ public class DefaultTransportService implements TransportService {
ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg))
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build();
ruleEngineMsgProducer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback);
ruleEngineProducerStats.incrementTotal();
StatsCallback wrappedCallback = new StatsCallback(callback, ruleEngineProducerStats);
ruleEngineMsgProducer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), wrappedCallback);
}
private class TransportTbQueueCallback implements TbQueueCallback {
@ -592,6 +611,30 @@ public class DefaultTransportService implements TransportService {
}
}
private class StatsCallback implements TbQueueCallback {
private final TbQueueCallback callback;
private final MessagesStats stats;
private StatsCallback(TbQueueCallback callback, MessagesStats stats) {
this.callback = callback;
this.stats = stats;
}
@Override
public void onSuccess(TbQueueMsgMetadata metadata) {
stats.incrementSuccessful();
if (callback != null)
callback.onSuccess(metadata);
}
@Override
public void onFailure(Throwable t) {
stats.incrementFailed();
if (callback != null)
callback.onFailure(t);
}
}
private class MsgPackCallback implements TbQueueCallback {
private final AtomicInteger msgCount;
private final TransportServiceCallback<Void> callback;

99
common/util/src/main/java/org/thingsboard/common/util/AzureIotHubUtil.java

@ -0,0 +1,99 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.common.util;
import lombok.extern.slf4j.Slf4j;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.io.IOException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Base64;
@Slf4j
public final class AzureIotHubUtil {
private static final String BASE_DIR_PATH = System.getProperty("user.dir");
private static final String APP_DIR = "application";
private static final String SRC_DIR = "src";
private static final String MAIN_DIR = "main";
private static final String DATA_DIR = "data";
private static final String CERTS_DIR = "certs";
private static final String AZURE_DIR = "azure";
private static final String FILE_NAME = "BaltimoreCyberTrustRoot.crt.pem";
private static final Path FULL_FILE_PATH;
static {
if (BASE_DIR_PATH.endsWith("bin")) {
FULL_FILE_PATH = Paths.get(BASE_DIR_PATH.replaceAll("bin$", ""), DATA_DIR, CERTS_DIR, AZURE_DIR, FILE_NAME);
} else if (BASE_DIR_PATH.endsWith("conf")) {
FULL_FILE_PATH = Paths.get(BASE_DIR_PATH.replaceAll("conf$", ""), DATA_DIR, CERTS_DIR, AZURE_DIR, FILE_NAME);
} else {
FULL_FILE_PATH = Paths.get(BASE_DIR_PATH, APP_DIR, SRC_DIR, MAIN_DIR, DATA_DIR, CERTS_DIR, AZURE_DIR, FILE_NAME);
}
}
private static final long SAS_TOKEN_VALID_SECS = 365 * 24 * 60 * 60;
private static final long ONE_SECOND_IN_MILLISECONDS = 1000;
private static final String SAS_TOKEN_FORMAT = "SharedAccessSignature sr=%s&sig=%s&se=%s";
private static final String USERNAME_FORMAT = "%s/%s/?api-version=2018-06-30";
private AzureIotHubUtil() {
}
public static String buildUsername(String host, String deviceId) {
return String.format(USERNAME_FORMAT, host, deviceId);
}
public static String buildSasToken(String host, String sasKey) {
try {
final String targetUri = URLEncoder.encode(host.toLowerCase(), "UTF-8");
final long expiryTime = buildExpiresOn();
String toSign = targetUri + "\n" + expiryTime;
byte[] keyBytes = Base64.getDecoder().decode(sasKey.getBytes(StandardCharsets.UTF_8));
SecretKeySpec signingKey = new SecretKeySpec(keyBytes, "HmacSHA256");
Mac mac = Mac.getInstance("HmacSHA256");
mac.init(signingKey);
byte[] rawHmac = mac.doFinal(toSign.getBytes(StandardCharsets.UTF_8));
String signature = URLEncoder.encode(Base64.getEncoder().encodeToString(rawHmac), "UTF-8");
return String.format(SAS_TOKEN_FORMAT, targetUri, signature, expiryTime);
} catch (Exception e) {
throw new RuntimeException("Failed to build SAS token!!!", e);
}
}
private static long buildExpiresOn() {
long expiresOnDate = System.currentTimeMillis();
expiresOnDate += SAS_TOKEN_VALID_SECS * ONE_SECOND_IN_MILLISECONDS;
return expiresOnDate / ONE_SECOND_IN_MILLISECONDS;
}
public static String getDefaultCaCert() {
try {
return new String(Files.readAllBytes(FULL_FILE_PATH));
} catch (IOException e) {
log.error("Failed to load Default CaCert file!!! [{}]", FULL_FILE_PATH.toString());
throw new RuntimeException("Failed to load Default CaCert file!!!");
}
}
}

4
dao/pom.xml

@ -43,6 +43,10 @@
<groupId>org.thingsboard.common</groupId>
<artifactId>message</artifactId>
</dependency>
<dependency>
<groupId>org.thingsboard.common</groupId>
<artifactId>stats</artifactId>
</dependency>
<dependency>
<groupId>org.thingsboard.common</groupId>
<artifactId>dao-api</artifactId>

77
dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateExecutor.java

@ -24,6 +24,9 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.stats.DefaultCounter;
import org.thingsboard.server.common.stats.StatsCounter;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.util.AbstractBufferedRateExecutor;
import org.thingsboard.server.dao.util.AsyncTaskContext;
@ -57,48 +60,58 @@ public class CassandraBufferedRateExecutor extends AbstractBufferedRateExecutor<
@Value("${cassandra.query.tenant_rate_limits.enabled}") boolean tenantRateLimitsEnabled,
@Value("${cassandra.query.tenant_rate_limits.configuration}") String tenantRateLimitsConfiguration,
@Value("${cassandra.query.tenant_rate_limits.print_tenant_names}") boolean printTenantNames,
@Value("${cassandra.query.print_queries_freq:0}") int printQueriesFreq) {
super(queueLimit, concurrencyLimit, maxWaitTime, dispatcherThreads, callbackThreads, pollMs, tenantRateLimitsEnabled, tenantRateLimitsConfiguration, printQueriesFreq);
@Value("${cassandra.query.print_queries_freq:0}") int printQueriesFreq,
@Autowired StatsFactory statsFactory) {
super(queueLimit, concurrencyLimit, maxWaitTime, dispatcherThreads, callbackThreads, pollMs, tenantRateLimitsEnabled, tenantRateLimitsConfiguration, printQueriesFreq, statsFactory);
this.printTenantNames = printTenantNames;
}
@Scheduled(fixedDelayString = "${cassandra.query.rate_limit_print_interval_ms}")
public void printStats() {
int queueSize = getQueueSize();
int totalAddedValue = totalAdded.getAndSet(0);
int totalLaunchedValue = totalLaunched.getAndSet(0);
int totalReleasedValue = totalReleased.getAndSet(0);
int totalFailedValue = totalFailed.getAndSet(0);
int totalExpiredValue = totalExpired.getAndSet(0);
int totalRejectedValue = totalRejected.getAndSet(0);
int totalRateLimitedValue = totalRateLimited.getAndSet(0);
int rateLimitedTenantsValue = rateLimitedTenants.size();
int concurrencyLevelValue = concurrencyLevel.get();
if (queueSize > 0 || totalAddedValue > 0 || totalLaunchedValue > 0 || totalReleasedValue > 0 ||
totalFailedValue > 0 || totalExpiredValue > 0 || totalRejectedValue > 0 || totalRateLimitedValue > 0 || rateLimitedTenantsValue > 0
|| concurrencyLevelValue > 0) {
log.info("Permits queueSize [{}] totalAdded [{}] totalLaunched [{}] totalReleased [{}] totalFailed [{}] totalExpired [{}] totalRejected [{}] " +
"totalRateLimited [{}] totalRateLimitedTenants [{}] currBuffer [{}] ",
queueSize, totalAddedValue, totalLaunchedValue, totalReleasedValue,
totalFailedValue, totalExpiredValue, totalRejectedValue, totalRateLimitedValue, rateLimitedTenantsValue, concurrencyLevelValue);
int rateLimitedTenantsCount = (int) stats.getRateLimitedTenants().values().stream()
.filter(defaultCounter -> defaultCounter.get() > 0)
.count();
if (queueSize > 0
|| rateLimitedTenantsCount > 0
|| concurrencyLevel.get() > 0
|| stats.getStatsCounters().stream().anyMatch(counter -> counter.get() > 0)
) {
StringBuilder statsBuilder = new StringBuilder();
statsBuilder.append("queueSize").append(" = [").append(queueSize).append("] ");
stats.getStatsCounters().forEach(counter -> {
statsBuilder.append(counter.getName()).append(" = [").append(counter.get()).append("] ");
});
statsBuilder.append("totalRateLimitedTenants").append(" = [").append(rateLimitedTenantsCount).append("] ");
statsBuilder.append(CONCURRENCY_LEVEL).append(" = [").append(concurrencyLevel.get()).append("] ");
stats.getStatsCounters().forEach(StatsCounter::clear);
log.info("Permits {}", statsBuilder);
}
rateLimitedTenants.forEach(((tenantId, counter) -> {
if (printTenantNames) {
String name = tenantNamesCache.computeIfAbsent(tenantId, tId -> {
try {
return entityService.fetchEntityNameAsync(TenantId.SYS_TENANT_ID, tenantId).get();
} catch (Exception e) {
log.error("[{}] Failed to get tenant name", tenantId, e);
return "N/A";
stats.getRateLimitedTenants().entrySet().stream()
.filter(entry -> entry.getValue().get() > 0)
.forEach(entry -> {
TenantId tenantId = entry.getKey();
DefaultCounter counter = entry.getValue();
int rateLimitedRequests = counter.get();
counter.clear();
if (printTenantNames) {
String name = tenantNamesCache.computeIfAbsent(tenantId, tId -> {
try {
return entityService.fetchEntityNameAsync(TenantId.SYS_TENANT_ID, tenantId).get();
} catch (Exception e) {
log.error("[{}] Failed to get tenant name", tenantId, e);
return "N/A";
}
});
log.info("[{}][{}] Rate limited requests: {}", tenantId, name, rateLimitedRequests);
} else {
log.info("[{}] Rate limited requests: {}", tenantId, rateLimitedRequests);
}
});
log.info("[{}][{}] Rate limited requests: {}", tenantId, name, counter);
} else {
log.info("[{}] Rate limited requests: {}", tenantId, counter);
}
}));
rateLimitedTenants.clear();
}
@PreDestroy

4
dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java

@ -31,7 +31,7 @@ import java.util.regex.Pattern;
@Slf4j
public abstract class DataValidator<D extends BaseData<?>> {
private static final Pattern EMAIL_PATTERN =
Pattern.compile("^[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,6}$", Pattern.CASE_INSENSITIVE);
Pattern.compile("^[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}$", Pattern.CASE_INSENSITIVE);
public void validate(D data, Function<D, TenantId> tenantIdFunction) {
try {
@ -64,7 +64,7 @@ public abstract class DataValidator<D extends BaseData<?>> {
return actualData.getId() != null && existentData.getId().equals(actualData.getId());
}
protected static void validateEmail(String email) {
public static void validateEmail(String email) {
if (!doValidateEmail(email)) {
throw new DataValidationException("Invalid email address format '" + email + "'!");
}

28
dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueue.java

@ -19,6 +19,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.stats.MessagesStats;
import java.util.ArrayList;
import java.util.List;
@ -27,7 +28,6 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@ -35,22 +35,19 @@ import java.util.stream.Collectors;
public class TbSqlBlockingQueue<E> implements TbSqlQueue<E> {
private final BlockingQueue<TbSqlQueueElement<E>> queue = new LinkedBlockingQueue<>();
private final AtomicInteger addedCount = new AtomicInteger();
private final AtomicInteger savedCount = new AtomicInteger();
private final AtomicInteger failedCount = new AtomicInteger();
private final TbSqlBlockingQueueParams params;
private ExecutorService executor;
private ScheduledLogExecutorComponent logExecutor;
private final MessagesStats stats;
public TbSqlBlockingQueue(TbSqlBlockingQueueParams params) {
public TbSqlBlockingQueue(TbSqlBlockingQueueParams params, MessagesStats stats) {
this.params = params;
this.stats = stats;
}
@Override
public void init(ScheduledLogExecutorComponent logExecutor, Consumer<List<E>> saveFunction) {
this.logExecutor = logExecutor;
executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("sql-queue-" + params.getLogName().toLowerCase()));
public void init(ScheduledLogExecutorComponent logExecutor, Consumer<List<E>> saveFunction, int index) {
executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("sql-queue-" + index + "-" + params.getLogName().toLowerCase()));
executor.submit(() -> {
String logName = params.getLogName();
int batchSize = params.getBatchSize();
@ -70,7 +67,7 @@ public class TbSqlBlockingQueue<E> implements TbSqlQueue<E> {
log.debug("[{}] Going to save {} entities", logName, entities.size());
saveFunction.accept(entities.stream().map(TbSqlQueueElement::getEntity).collect(Collectors.toList()));
entities.forEach(v -> v.getFuture().set(null));
savedCount.addAndGet(entities.size());
stats.incrementSuccessful(entities.size());
if (!fullPack) {
long remainingDelay = maxDelay - (System.currentTimeMillis() - currentTs);
if (remainingDelay > 0) {
@ -78,7 +75,7 @@ public class TbSqlBlockingQueue<E> implements TbSqlQueue<E> {
}
}
} catch (Exception e) {
failedCount.addAndGet(entities.size());
stats.incrementFailed(entities.size());
entities.forEach(entityFutureWrapper -> entityFutureWrapper.getFuture().setException(e));
if (e instanceof InterruptedException) {
log.info("[{}] Queue polling was interrupted", logName);
@ -93,9 +90,10 @@ public class TbSqlBlockingQueue<E> implements TbSqlQueue<E> {
});
logExecutor.scheduleAtFixedRate(() -> {
if (queue.size() > 0 || addedCount.get() > 0 || savedCount.get() > 0 || failedCount.get() > 0) {
log.info("[{}] queueSize [{}] totalAdded [{}] totalSaved [{}] totalFailed [{}]",
params.getLogName(), queue.size(), addedCount.getAndSet(0), savedCount.getAndSet(0), failedCount.getAndSet(0));
if (queue.size() > 0 || stats.getTotal() > 0 || stats.getSuccessful() > 0 || stats.getFailed() > 0) {
log.info("Queue-{} [{}] queueSize [{}] totalAdded [{}] totalSaved [{}] totalFailed [{}]", index,
params.getLogName(), queue.size(), stats.getTotal(), stats.getSuccessful(), stats.getFailed());
stats.reset();
}
}, params.getStatsPrintIntervalMs(), params.getStatsPrintIntervalMs(), TimeUnit.MILLISECONDS);
}
@ -111,7 +109,7 @@ public class TbSqlBlockingQueue<E> implements TbSqlQueue<E> {
public ListenableFuture<Void> add(E element) {
SettableFuture<Void> future = SettableFuture.create();
queue.add(new TbSqlQueueElement<>(future, element));
addedCount.incrementAndGet();
stats.incrementTotal();
return future;
}
}

3
dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueueParams.java

@ -18,6 +18,8 @@ package org.thingsboard.server.dao.sql;
import lombok.Builder;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.stats.MessagesStats;
import org.thingsboard.server.common.stats.StatsFactory;
@Slf4j
@Data
@ -28,4 +30,5 @@ public class TbSqlBlockingQueueParams {
private final int batchSize;
private final long maxDelay;
private final long statsPrintIntervalMs;
private final String statsNamePrefix;
}

56
dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueueWrapper.java

@ -0,0 +1,56 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sql;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.stats.MessagesStats;
import org.thingsboard.server.common.stats.StatsFactory;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.function.Consumer;
import java.util.function.Function;
@Slf4j
@Data
public class TbSqlBlockingQueueWrapper<E> {
private final CopyOnWriteArrayList<TbSqlBlockingQueue<E>> queues = new CopyOnWriteArrayList<>();
private final TbSqlBlockingQueueParams params;
private ScheduledLogExecutorComponent logExecutor;
private final Function<E, Integer> hashCodeFunction;
private final int maxThreads;
private final StatsFactory statsFactory;
public void init(ScheduledLogExecutorComponent logExecutor, Consumer<List<E>> saveFunction) {
for (int i = 0; i < maxThreads; i++) {
MessagesStats stats = statsFactory.createMessagesStats(params.getStatsNamePrefix() + ".queue." + i);
TbSqlBlockingQueue<E> queue = new TbSqlBlockingQueue<>(params, stats);
queues.add(queue);
queue.init(logExecutor, saveFunction, i);
}
}
public ListenableFuture<Void> add(E element) {
int queueIndex = element != null ? (hashCodeFunction.apply(element) & 0x7FFFFFFF) % maxThreads : 0;
return queues.get(queueIndex).add(element);
}
public void destroy() {
queues.forEach(TbSqlBlockingQueue::destroy);
}
}

2
dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlQueue.java

@ -22,7 +22,7 @@ import java.util.function.Consumer;
public interface TbSqlQueue<E> {
void init(ScheduledLogExecutorComponent logExecutor, Consumer<List<E>> saveFunction);
void init(ScheduledLogExecutorComponent logExecutor, Consumer<List<E>> saveFunction, int queueIndex);
void destroy();

17
dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java

@ -26,14 +26,15 @@ import org.thingsboard.server.common.data.UUIDConverter;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.attributes.AttributesDao;
import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey;
import org.thingsboard.server.dao.model.sql.AttributeKvEntity;
import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService;
import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueue;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper;
import org.thingsboard.server.dao.util.SqlDao;
import javax.annotation.PostConstruct;
@ -41,6 +42,7 @@ import javax.annotation.PreDestroy;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID;
@ -59,6 +61,9 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl
@Autowired
private AttributeKvInsertRepository attributeKvInsertRepository;
@Autowired
private StatsFactory statsFactory;
@Value("${sql.attributes.batch_size:1000}")
private int batchSize;
@ -68,7 +73,10 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl
@Value("${sql.attributes.stats_print_interval_ms:1000}")
private long statsPrintIntervalMs;
private TbSqlBlockingQueue<AttributeKvEntity> queue;
@Value("${sql.attributes.batch_threads:4}")
private int batchThreads;
private TbSqlBlockingQueueWrapper<AttributeKvEntity> queue;
@PostConstruct
private void init() {
@ -77,8 +85,11 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl
.batchSize(batchSize)
.maxDelay(maxDelay)
.statsPrintIntervalMs(statsPrintIntervalMs)
.statsNamePrefix("attributes")
.build();
queue = new TbSqlBlockingQueue<>(params);
Function<AttributeKvEntity, Integer> hashcodeFunction = entity -> entity.getId().getEntityId().hashCode();
queue = new TbSqlBlockingQueueWrapper<>(params, hashcodeFunction, batchThreads, statsFactory);
queue.init(logExecutor, v -> attributeKvInsertRepository.saveOrUpdate(v));
}

13
dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java

@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sqlts;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.SettableFuture;
import lombok.extern.slf4j.Slf4j;
@ -29,10 +30,12 @@ import org.thingsboard.server.common.data.kv.Aggregation;
import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueue;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper;
import org.thingsboard.server.dao.sqlts.insert.InsertTsRepository;
import org.thingsboard.server.dao.sqlts.ts.TsKvRepository;
import org.thingsboard.server.dao.timeseries.TimeseriesDao;
@ -43,6 +46,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.function.Function;
import java.util.stream.Collectors;
@Slf4j
@ -54,7 +58,9 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
@Autowired
protected InsertTsRepository<TsKvEntity> insertRepository;
protected TbSqlBlockingQueue<TsKvEntity> tsQueue;
protected TbSqlBlockingQueueWrapper<TsKvEntity> tsQueue;
@Autowired
private StatsFactory statsFactory;
@PostConstruct
protected void init() {
@ -64,8 +70,11 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
.batchSize(tsBatchSize)
.maxDelay(tsMaxDelay)
.statsPrintIntervalMs(tsStatsPrintIntervalMs)
.statsNamePrefix("ts")
.build();
tsQueue = new TbSqlBlockingQueue<>(tsParams);
Function<TsKvEntity, Integer> hashcodeFunction = entity -> entity.getEntityId().hashCode();
tsQueue = new TbSqlBlockingQueueWrapper<>(tsParams, hashcodeFunction, tsBatchThreads, statsFactory);
tsQueue.init(logExecutor, v -> insertRepository.saveOrUpdate(v));
}

39
dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java

@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
import org.thingsboard.server.common.data.kv.StringDataEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary;
import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey;
@ -41,8 +42,8 @@ import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestCompositeKey;
import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity;
import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService;
import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueue;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper;
import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository;
import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository;
import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository;
@ -52,7 +53,10 @@ import org.thingsboard.server.dao.timeseries.SimpleListenableFuture;
import javax.annotation.Nullable;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
@ -82,7 +86,7 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
@Autowired
private TsKvDictionaryRepository dictionaryRepository;
private TbSqlBlockingQueue<TsKvLatestEntity> tsLatestQueue;
private TbSqlBlockingQueueWrapper<TsKvLatestEntity> tsLatestQueue;
@Value("${sql.ts_latest.batch_size:1000}")
private int tsLatestBatchSize;
@ -93,9 +97,15 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
@Value("${sql.ts_latest.stats_print_interval_ms:1000}")
private long tsLatestStatsPrintIntervalMs;
@Value("${sql.ts_latest.batch_threads:4}")
private int tsLatestBatchThreads;
@Autowired
protected ScheduledLogExecutorComponent logExecutor;
@Autowired
private StatsFactory statsFactory;
@Value("${sql.ts.batch_size:1000}")
protected int tsBatchSize;
@ -105,6 +115,12 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
@Value("${sql.ts.stats_print_interval_ms:1000}")
protected long tsStatsPrintIntervalMs;
@Value("${sql.ts.batch_threads:4}")
protected int tsBatchThreads;
@Value("${sql.timescale.batch_threads:4}")
protected int timescaleBatchThreads;
@PostConstruct
protected void init() {
TbSqlBlockingQueueParams tsLatestParams = TbSqlBlockingQueueParams.builder()
@ -112,9 +128,24 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
.batchSize(tsLatestBatchSize)
.maxDelay(tsLatestMaxDelay)
.statsPrintIntervalMs(tsLatestStatsPrintIntervalMs)
.statsNamePrefix("ts.latest")
.build();
tsLatestQueue = new TbSqlBlockingQueue<>(tsLatestParams);
tsLatestQueue.init(logExecutor, v -> insertLatestTsRepository.saveOrUpdate(v));
java.util.function.Function<TsKvLatestEntity, Integer> hashcodeFunction = entity -> entity.getEntityId().hashCode();
tsLatestQueue = new TbSqlBlockingQueueWrapper<>(tsLatestParams, hashcodeFunction, tsLatestBatchThreads, statsFactory);
tsLatestQueue.init(logExecutor, v -> {
Map<TsKey, TsKvLatestEntity> trueLatest = new HashMap<>();
v.forEach(ts -> {
TsKey key = new TsKey(ts.getEntityId(), ts.getKey());
TsKvLatestEntity old = trueLatest.get(key);
if (old == null || old.getTs() < ts.getTs()) {
trueLatest.put(key, ts);
}
});
List<TsKvLatestEntity> latestEntities = new ArrayList<>(trueLatest.values());
insertLatestTsRepository.saveOrUpdate(latestEntities);
});
}
@PreDestroy

26
dao/src/main/java/org/thingsboard/server/dao/sqlts/TsKey.java

@ -0,0 +1,26 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sqlts;
import lombok.Data;
import java.util.UUID;
@Data
public class TsKey {
private final UUID entityId;
private final int key;
}

16
dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java

@ -31,10 +31,11 @@ import org.thingsboard.server.common.data.kv.Aggregation;
import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueue;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper;
import org.thingsboard.server.dao.sqlts.AbstractSqlTimeseriesDao;
import org.thingsboard.server.dao.sqlts.insert.InsertTsRepository;
import org.thingsboard.server.dao.timeseries.TimeseriesDao;
@ -48,6 +49,7 @@ import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.function.Function;
@Component
@Slf4j
@ -60,10 +62,13 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
@Autowired
private AggregationRepository aggregationRepository;
@Autowired
private StatsFactory statsFactory;
@Autowired
protected InsertTsRepository<TimescaleTsKvEntity> insertRepository;
protected TbSqlBlockingQueue<TimescaleTsKvEntity> tsQueue;
protected TbSqlBlockingQueueWrapper<TimescaleTsKvEntity> tsQueue;
@PostConstruct
protected void init() {
@ -73,8 +78,12 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
.batchSize(tsBatchSize)
.maxDelay(tsMaxDelay)
.statsPrintIntervalMs(tsStatsPrintIntervalMs)
.statsNamePrefix("ts.timescale")
.build();
tsQueue = new TbSqlBlockingQueue<>(tsParams);
Function<TimescaleTsKvEntity, Integer> hashcodeFunction = entity -> entity.getEntityId().hashCode();
tsQueue = new TbSqlBlockingQueueWrapper<>(tsParams, hashcodeFunction, timescaleBatchThreads, statsFactory);
tsQueue.init(logExecutor, v -> insertRepository.saveOrUpdate(v));
}
@ -277,4 +286,5 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
startTs,
endTs);
}
}

44
dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java

@ -23,6 +23,8 @@ import com.google.common.util.concurrent.SettableFuture;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.stats.StatsType;
import org.thingsboard.server.common.msg.tools.TbRateLimits;
import org.thingsboard.server.dao.nosql.CassandraStatementTask;
@ -38,6 +40,8 @@ import java.util.regex.Matcher;
@Slf4j
public abstract class AbstractBufferedRateExecutor<T extends AsyncTask, F extends ListenableFuture<V>, V> implements BufferedRateExecutor<T, F> {
public static final String CONCURRENCY_LEVEL = "currBuffer";
private final long maxWaitTime;
private final long pollMs;
private final BlockingQueue<AsyncTaskContext<T, V>> queue;
@ -49,20 +53,14 @@ public abstract class AbstractBufferedRateExecutor<T extends AsyncTask, F extend
private final boolean perTenantLimitsEnabled;
private final String perTenantLimitsConfiguration;
private final ConcurrentMap<TenantId, TbRateLimits> perTenantLimits = new ConcurrentHashMap<>();
protected final ConcurrentMap<TenantId, AtomicInteger> rateLimitedTenants = new ConcurrentHashMap<>();
protected final AtomicInteger concurrencyLevel = new AtomicInteger();
protected final AtomicInteger totalAdded = new AtomicInteger();
protected final AtomicInteger totalLaunched = new AtomicInteger();
protected final AtomicInteger totalReleased = new AtomicInteger();
protected final AtomicInteger totalFailed = new AtomicInteger();
protected final AtomicInteger totalExpired = new AtomicInteger();
protected final AtomicInteger totalRejected = new AtomicInteger();
protected final AtomicInteger totalRateLimited = new AtomicInteger();
protected final AtomicInteger printQueriesIdx = new AtomicInteger();
private final AtomicInteger printQueriesIdx = new AtomicInteger(0);
protected final AtomicInteger concurrencyLevel;
protected final BufferedRateExecutorStats stats;
public AbstractBufferedRateExecutor(int queueLimit, int concurrencyLimit, long maxWaitTime, int dispatcherThreads, int callbackThreads, long pollMs,
boolean perTenantLimitsEnabled, String perTenantLimitsConfiguration, int printQueriesFreq) {
boolean perTenantLimitsEnabled, String perTenantLimitsConfiguration, int printQueriesFreq, StatsFactory statsFactory) {
this.maxWaitTime = maxWaitTime;
this.pollMs = pollMs;
this.concurrencyLimit = concurrencyLimit;
@ -73,6 +71,10 @@ public abstract class AbstractBufferedRateExecutor<T extends AsyncTask, F extend
this.timeoutExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("nosql-timeout"));
this.perTenantLimitsEnabled = perTenantLimitsEnabled;
this.perTenantLimitsConfiguration = perTenantLimitsConfiguration;
this.stats = new BufferedRateExecutorStats(statsFactory);
String concurrencyLevelKey = StatsType.RATE_EXECUTOR.getName() + "." + CONCURRENCY_LEVEL;
this.concurrencyLevel = statsFactory.createGauge(concurrencyLevelKey, new AtomicInteger(0));
for (int i = 0; i < dispatcherThreads; i++) {
dispatcherExecutor.submit(this::dispatch);
}
@ -89,8 +91,8 @@ public abstract class AbstractBufferedRateExecutor<T extends AsyncTask, F extend
} else if (!task.getTenantId().isNullUid()) {
TbRateLimits rateLimits = perTenantLimits.computeIfAbsent(task.getTenantId(), id -> new TbRateLimits(perTenantLimitsConfiguration));
if (!rateLimits.tryConsume()) {
rateLimitedTenants.computeIfAbsent(task.getTenantId(), tId -> new AtomicInteger(0)).incrementAndGet();
totalRateLimited.incrementAndGet();
stats.incrementRateLimitedTenant(task.getTenantId());
stats.getTotalRateLimited().increment();
settableFuture.setException(new TenantRateLimitException());
perTenantLimitReached = true;
}
@ -98,10 +100,10 @@ public abstract class AbstractBufferedRateExecutor<T extends AsyncTask, F extend
}
if (!perTenantLimitReached) {
try {
totalAdded.incrementAndGet();
stats.getTotalAdded().increment();
queue.add(new AsyncTaskContext<>(UUID.randomUUID(), task, settableFuture, System.currentTimeMillis()));
} catch (IllegalStateException e) {
totalRejected.incrementAndGet();
stats.getTotalRejected().increment();
settableFuture.setException(e);
}
}
@ -146,14 +148,14 @@ public abstract class AbstractBufferedRateExecutor<T extends AsyncTask, F extend
concurrencyLevel.incrementAndGet();
long timeout = finalTaskCtx.getCreateTime() + maxWaitTime - System.currentTimeMillis();
if (timeout > 0) {
totalLaunched.incrementAndGet();
stats.getTotalLaunched().increment();
ListenableFuture<V> result = execute(finalTaskCtx);
result = Futures.withTimeout(result, timeout, TimeUnit.MILLISECONDS, timeoutExecutor);
Futures.addCallback(result, new FutureCallback<V>() {
@Override
public void onSuccess(@Nullable V result) {
logTask("Releasing", finalTaskCtx);
totalReleased.incrementAndGet();
stats.getTotalReleased().increment();
concurrencyLevel.decrementAndGet();
finalTaskCtx.getFuture().set(result);
}
@ -165,7 +167,7 @@ public abstract class AbstractBufferedRateExecutor<T extends AsyncTask, F extend
} else {
logTask("Failed", finalTaskCtx);
}
totalFailed.incrementAndGet();
stats.getTotalFailed().increment();
concurrencyLevel.decrementAndGet();
finalTaskCtx.getFuture().setException(t);
log.debug("[{}] Failed to execute task: {}", finalTaskCtx.getId(), finalTaskCtx.getTask(), t);
@ -173,7 +175,7 @@ public abstract class AbstractBufferedRateExecutor<T extends AsyncTask, F extend
}, callbackExecutor);
} else {
logTask("Expired Before Execution", finalTaskCtx);
totalExpired.incrementAndGet();
stats.getTotalExpired().increment();
concurrencyLevel.decrementAndGet();
taskCtx.getFuture().setException(new TimeoutException());
}
@ -185,7 +187,7 @@ public abstract class AbstractBufferedRateExecutor<T extends AsyncTask, F extend
} catch (Throwable e) {
if (taskCtx != null) {
log.debug("[{}] Failed to execute task: {}", taskCtx.getId(), taskCtx, e);
totalFailed.incrementAndGet();
stats.getTotalFailed().increment();
concurrencyLevel.decrementAndGet();
} else {
log.debug("Failed to queue task:", e);

90
dao/src/main/java/org/thingsboard/server/dao/util/BufferedRateExecutorStats.java

@ -0,0 +1,90 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.util;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.stats.DefaultCounter;
import org.thingsboard.server.common.stats.StatsCounter;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.stats.StatsType;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@Slf4j
@Getter
public class BufferedRateExecutorStats {
private static final String TENANT_ID_TAG = "tenantId";
private static final String TOTAL_ADDED = "totalAdded";
private static final String TOTAL_LAUNCHED = "totalLaunched";
private static final String TOTAL_RELEASED = "totalReleased";
private static final String TOTAL_FAILED = "totalFailed";
private static final String TOTAL_EXPIRED = "totalExpired";
private static final String TOTAL_REJECTED = "totalRejected";
private static final String TOTAL_RATE_LIMITED = "totalRateLimited";
private final StatsFactory statsFactory;
private final ConcurrentMap<TenantId, DefaultCounter> rateLimitedTenants = new ConcurrentHashMap<>();
private final List<StatsCounter> statsCounters = new ArrayList<>();
private final StatsCounter totalAdded;
private final StatsCounter totalLaunched;
private final StatsCounter totalReleased;
private final StatsCounter totalFailed;
private final StatsCounter totalExpired;
private final StatsCounter totalRejected;
private final StatsCounter totalRateLimited;
public BufferedRateExecutorStats(StatsFactory statsFactory) {
this.statsFactory = statsFactory;
String key = StatsType.RATE_EXECUTOR.getName();
this.totalAdded = statsFactory.createStatsCounter(key, TOTAL_ADDED);
this.totalLaunched = statsFactory.createStatsCounter(key, TOTAL_LAUNCHED);
this.totalReleased = statsFactory.createStatsCounter(key, TOTAL_RELEASED);
this.totalFailed = statsFactory.createStatsCounter(key, TOTAL_FAILED);
this.totalExpired = statsFactory.createStatsCounter(key, TOTAL_EXPIRED);
this.totalRejected = statsFactory.createStatsCounter(key, TOTAL_REJECTED);
this.totalRateLimited = statsFactory.createStatsCounter(key, TOTAL_RATE_LIMITED);
this.statsCounters.add(totalAdded);
this.statsCounters.add(totalLaunched);
this.statsCounters.add(totalReleased);
this.statsCounters.add(totalFailed);
this.statsCounters.add(totalExpired);
this.statsCounters.add(totalRejected);
this.statsCounters.add(totalRateLimited);
}
public void incrementRateLimitedTenant(TenantId tenantId){
rateLimitedTenants.computeIfAbsent(tenantId,
tId -> {
String key = StatsType.RATE_EXECUTOR.getName() + ".tenant";
return statsFactory.createDefaultCounter(key, TENANT_ID_TAG, tId.toString());
}
)
.increment();
}
}

21
pom.xml

@ -105,6 +105,7 @@
<ua-parser.version>1.4.3</ua-parser.version>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
<commons-collections.version>3.2.2</commons-collections.version>
<micrometer.version>1.5.2</micrometer.version>
</properties>
<modules>
@ -842,6 +843,11 @@
<artifactId>queue</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.thingsboard.common</groupId>
<artifactId>stats</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.thingsboard</groupId>
<artifactId>tools</artifactId>
@ -1376,6 +1382,21 @@
<artifactId>struts-tiles</artifactId>
<version>${struts.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
<version>${spring-boot.version}</version>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId>
<version>${micrometer.version}</version>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
<version>${micrometer.version}</version>
</dependency>
</dependencies>
</dependencyManagement>

2
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java

@ -40,7 +40,7 @@ import org.thingsboard.server.common.msg.TbMsg;
nodeDetails =
"Details - JS function that creates JSON object based on incoming message. This object will be added into Alarm.details field.\n" +
"Node output:\n" +
"If alarm was not cleared, original message is returned. Otherwise new Message returned with type 'ALARM', Alarm object in 'msg' property and 'matadata' will contains 'isClearedAlarm' property. " +
"If alarm was not cleared, original message is returned. Otherwise new Message returned with type 'ALARM', Alarm object in 'msg' property and 'metadata' will contains 'isClearedAlarm' property. " +
"Message payload can be accessed via <code>msg</code> property. For example <code>'temperature = ' + msg.temperature ;</code>. " +
"Message metadata can be accessed via <code>metadata</code> property. For example <code>'name = ' + metadata.customerName;</code>.",
uiResources = {"static/rulenode/rulenode-core-config.js"},

2
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCreateAlarmNode.java

@ -46,7 +46,7 @@ import java.util.List;
nodeDetails =
"Details - JS function that creates JSON object based on incoming message. This object will be added into Alarm.details field.\n" +
"Node output:\n" +
"If alarm was not created, original message is returned. Otherwise new Message returned with type 'ALARM', Alarm object in 'msg' property and 'matadata' will contains one of those properties 'isNewAlarm/isExistingAlarm'. " +
"If alarm was not created, original message is returned. Otherwise new Message returned with type 'ALARM', Alarm object in 'msg' property and 'metadata' will contains one of those properties 'isNewAlarm/isExistingAlarm'. " +
"Message payload can be accessed via <code>msg</code> property. For example <code>'temperature = ' + msg.temperature ;</code>. " +
"Message metadata can be accessed via <code>metadata</code> property. For example <code>'name = ' + metadata.customerName;</code>.",
uiResources = {"static/rulenode/rulenode-core-config.js"},

2
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java

@ -67,7 +67,7 @@ public class TbCheckAlarmStatusNode implements TbNode {
if (result != null) {
boolean isPresent = false;
for (AlarmStatus alarmStatus : config.getAlarmStatusList()) {
if (alarm.getStatus() == alarmStatus) {
if (result.getStatus() == alarmStatus) {
isPresent = true;
break;
}

33
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java

@ -16,8 +16,6 @@
package org.thingsboard.rule.engine.mqtt;
import io.netty.buffer.Unpooled;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.handler.codec.mqtt.MqttQoS;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
@ -37,7 +35,6 @@ import org.thingsboard.server.common.msg.TbMsgMetaData;
import javax.net.ssl.SSLException;
import java.nio.charset.Charset;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@ -58,14 +55,14 @@ public class TbMqttNode implements TbNode {
private static final String ERROR = "error";
private TbMqttNodeConfiguration config;
protected TbMqttNodeConfiguration mqttNodeConfiguration;
private MqttClient mqttClient;
protected MqttClient mqttClient;
@Override
public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException {
try {
this.config = TbNodeUtils.convert(configuration, TbMqttNodeConfiguration.class);
this.mqttNodeConfiguration = TbNodeUtils.convert(configuration, TbMqttNodeConfiguration.class);
this.mqttClient = initClient(ctx);
} catch (Exception e) {
throw new TbNodeException(e);
@ -74,7 +71,7 @@ public class TbMqttNode implements TbNode {
@Override
public void onMsg(TbContext ctx, TbMsg msg) {
String topic = TbNodeUtils.processPattern(this.config.getTopicPattern(), msg.getMetaData());
String topic = TbNodeUtils.processPattern(this.mqttNodeConfiguration.getTopicPattern(), msg.getMetaData());
this.mqttClient.publish(topic, Unpooled.wrappedBuffer(msg.getData().getBytes(UTF8)), MqttQoS.AT_LEAST_ONCE)
.addListener(future -> {
if (future.isSuccess()) {
@ -100,38 +97,38 @@ public class TbMqttNode implements TbNode {
}
}
private MqttClient initClient(TbContext ctx) throws Exception {
protected MqttClient initClient(TbContext ctx) throws Exception {
Optional<SslContext> sslContextOpt = initSslContext();
MqttClientConfig config = sslContextOpt.isPresent() ? new MqttClientConfig(sslContextOpt.get()) : new MqttClientConfig();
if (!StringUtils.isEmpty(this.config.getClientId())) {
config.setClientId(this.config.getClientId());
if (!StringUtils.isEmpty(this.mqttNodeConfiguration.getClientId())) {
config.setClientId(this.mqttNodeConfiguration.getClientId());
}
config.setCleanSession(this.config.isCleanSession());
this.config.getCredentials().configure(config);
config.setCleanSession(this.mqttNodeConfiguration.isCleanSession());
this.mqttNodeConfiguration.getCredentials().configure(config);
MqttClient client = MqttClient.create(config, null);
client.setEventLoop(ctx.getSharedEventLoop());
Future<MqttConnectResult> connectFuture = client.connect(this.config.getHost(), this.config.getPort());
Future<MqttConnectResult> connectFuture = client.connect(this.mqttNodeConfiguration.getHost(), this.mqttNodeConfiguration.getPort());
MqttConnectResult result;
try {
result = connectFuture.get(this.config.getConnectTimeoutSec(), TimeUnit.SECONDS);
result = connectFuture.get(this.mqttNodeConfiguration.getConnectTimeoutSec(), TimeUnit.SECONDS);
} catch (TimeoutException ex) {
connectFuture.cancel(true);
client.disconnect();
String hostPort = this.config.getHost() + ":" + this.config.getPort();
String hostPort = this.mqttNodeConfiguration.getHost() + ":" + this.mqttNodeConfiguration.getPort();
throw new RuntimeException(String.format("Failed to connect to MQTT broker at %s.", hostPort));
}
if (!result.isSuccess()) {
connectFuture.cancel(true);
client.disconnect();
String hostPort = this.config.getHost() + ":" + this.config.getPort();
String hostPort = this.mqttNodeConfiguration.getHost() + ":" + this.mqttNodeConfiguration.getPort();
throw new RuntimeException(String.format("Failed to connect to MQTT broker at %s. Result code is: %s", hostPort, result.getReturnCode()));
}
return client;
}
private Optional<SslContext> initSslContext() throws SSLException {
Optional<SslContext> result = this.config.getCredentials().initSslContext();
if (this.config.isSsl() && !result.isPresent()) {
Optional<SslContext> result = this.mqttNodeConfiguration.getCredentials().initSslContext();
if (this.mqttNodeConfiguration.isSsl() && !result.isPresent()) {
result = Optional.of(SslContextBuilder.forClient().build());
}
return result;

91
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/azure/AzureIotHubSasCredentials.java

@ -0,0 +1,91 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.rule.engine.mqtt.azure;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import io.netty.handler.ssl.ClientAuth;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.binary.Base64;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.thingsboard.common.util.AzureIotHubUtil;
import org.thingsboard.mqtt.MqttClientConfig;
import org.thingsboard.rule.engine.mqtt.credentials.MqttClientCredentials;
import javax.net.ssl.TrustManagerFactory;
import java.io.ByteArrayInputStream;
import java.security.KeyStore;
import java.security.Security;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.util.Optional;
@Data
@Slf4j
@JsonIgnoreProperties(ignoreUnknown = true)
public class AzureIotHubSasCredentials implements MqttClientCredentials {
private String sasKey;
private String caCert;
@Override
public Optional<SslContext> initSslContext() {
try {
Security.addProvider(new BouncyCastleProvider());
if (caCert == null || caCert.isEmpty()) {
caCert = AzureIotHubUtil.getDefaultCaCert();
}
return Optional.of(SslContextBuilder.forClient()
.trustManager(createAndInitTrustManagerFactory())
.clientAuth(ClientAuth.REQUIRE)
.build());
} catch (Exception e) {
log.error("[{}] Creating TLS factory failed!", caCert, e);
throw new RuntimeException("Creating TLS factory failed!", e);
}
}
@Override
public void configure(MqttClientConfig config) {
}
private TrustManagerFactory createAndInitTrustManagerFactory() throws Exception {
X509Certificate caCertHolder;
caCertHolder = readCertFile(caCert);
KeyStore caKeyStore = KeyStore.getInstance(KeyStore.getDefaultType());
caKeyStore.load(null, null);
caKeyStore.setCertificateEntry("caCert-cert", caCertHolder);
TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
trustManagerFactory.init(caKeyStore);
return trustManagerFactory;
}
private X509Certificate readCertFile(String fileContent) throws Exception {
X509Certificate certificate = null;
if (fileContent != null && !fileContent.trim().isEmpty()) {
fileContent = fileContent.replace("-----BEGIN CERTIFICATE-----", "")
.replace("-----END CERTIFICATE-----", "")
.replaceAll("\\s", "");
byte[] decoded = Base64.decodeBase64(fileContent);
CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
certificate = (X509Certificate) certFactory.generateCertificate(new ByteArrayInputStream(decoded));
}
return certificate;
}
}

86
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/azure/TbAzureIotHubNode.java

@ -0,0 +1,86 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.rule.engine.mqtt.azure;
import io.netty.handler.codec.mqtt.MqttVersion;
import io.netty.handler.ssl.SslContext;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.AzureIotHubUtil;
import org.thingsboard.mqtt.MqttClientConfig;
import org.thingsboard.rule.engine.api.RuleNode;
import org.thingsboard.rule.engine.api.TbContext;
import org.thingsboard.rule.engine.api.TbNodeConfiguration;
import org.thingsboard.rule.engine.api.TbNodeException;
import org.thingsboard.rule.engine.api.util.TbNodeUtils;
import org.thingsboard.rule.engine.mqtt.TbMqttNode;
import org.thingsboard.rule.engine.mqtt.TbMqttNodeConfiguration;
import org.thingsboard.rule.engine.mqtt.credentials.CertPemClientCredentials;
import org.thingsboard.rule.engine.mqtt.credentials.MqttClientCredentials;
import org.thingsboard.server.common.data.plugin.ComponentType;
import java.util.Optional;
@Slf4j
@RuleNode(
type = ComponentType.EXTERNAL,
name = "azure iot hub",
configClazz = TbAzureIotHubNodeConfiguration.class,
nodeDescription = "Publish messages to the Azure IoT Hub",
nodeDetails = "Will publish message payload to the Azure IoT Hub with QoS <b>AT_LEAST_ONCE</b>.",
uiResources = {"static/rulenode/rulenode-core-config.js", "static/rulenode/rulenode-core-config.css"},
configDirective = "tbActionNodeAzureIotHubConfig"
)
public class TbAzureIotHubNode extends TbMqttNode {
@Override
public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException {
try {
this.mqttNodeConfiguration = TbNodeUtils.convert(configuration, TbMqttNodeConfiguration.class);
mqttNodeConfiguration.setPort(8883);
mqttNodeConfiguration.setCleanSession(true);
MqttClientCredentials credentials = mqttNodeConfiguration.getCredentials();
mqttNodeConfiguration.setCredentials(new MqttClientCredentials() {
@Override
public Optional<SslContext> initSslContext() {
if (credentials instanceof AzureIotHubSasCredentials) {
AzureIotHubSasCredentials sasCredentials = (AzureIotHubSasCredentials) credentials;
if (sasCredentials.getCaCert() == null || sasCredentials.getCaCert().isEmpty()) {
sasCredentials.setCaCert(AzureIotHubUtil.getDefaultCaCert());
}
} else if (credentials instanceof CertPemClientCredentials) {
CertPemClientCredentials pemCredentials = (CertPemClientCredentials) credentials;
if (pemCredentials.getCaCert() == null || pemCredentials.getCaCert().isEmpty()) {
pemCredentials.setCaCert(AzureIotHubUtil.getDefaultCaCert());
}
}
return credentials.initSslContext();
}
@Override
public void configure(MqttClientConfig config) {
config.setProtocolVersion(MqttVersion.MQTT_3_1_1);
config.setUsername(AzureIotHubUtil.buildUsername(mqttNodeConfiguration.getHost(), config.getClientId()));
if (credentials instanceof AzureIotHubSasCredentials) {
AzureIotHubSasCredentials sasCredentials = (AzureIotHubSasCredentials) credentials;
config.setPassword(AzureIotHubUtil.buildSasToken(mqttNodeConfiguration.getHost(), sasCredentials.getSasKey()));
}
}
});
this.mqttClient = initClient(ctx);
} catch (Exception e) {
throw new TbNodeException(e);
} }
}

40
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/azure/TbAzureIotHubNodeConfiguration.java

@ -0,0 +1,40 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.rule.engine.mqtt.azure;
import lombok.Data;
import org.thingsboard.rule.engine.api.NodeConfiguration;
import org.thingsboard.rule.engine.mqtt.TbMqttNodeConfiguration;
import org.thingsboard.rule.engine.mqtt.credentials.AnonymousCredentials;
import org.thingsboard.rule.engine.mqtt.credentials.MqttClientCredentials;
@Data
public class TbAzureIotHubNodeConfiguration extends TbMqttNodeConfiguration {
@Override
public TbAzureIotHubNodeConfiguration defaultConfiguration() {
TbAzureIotHubNodeConfiguration configuration = new TbAzureIotHubNodeConfiguration();
configuration.setTopicPattern("devices/<device_id>/messages/events/");
configuration.setHost("<iot-hub-name>.azure-devices.net");
configuration.setPort(8883);
configuration.setConnectTimeoutSec(10);
configuration.setCleanSession(true);
configuration.setSsl(true);
configuration.setCredentials(new AzureIotHubSasCredentials());
return configuration;
}
}

2
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/credentials/CertPemClientCredentials.java

@ -163,7 +163,7 @@ public class CertPemClientCredentials implements MqttClientCredentials {
private KeySpec getKeySpec(byte[] encodedKey) throws Exception {
KeySpec keySpec;
if (password == null) {
if (password == null || password.isEmpty()) {
keySpec = new PKCS8EncodedKeySpec(encodedKey);
} else {
PBEKeySpec pbeKeySpec = new PBEKeySpec(password.toCharArray());

2
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/credentials/MqttClientCredentials.java

@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import io.netty.handler.ssl.SslContext;
import org.thingsboard.mqtt.MqttClientConfig;
import org.thingsboard.rule.engine.mqtt.azure.AzureIotHubSasCredentials;
import java.util.Optional;
@ -29,6 +30,7 @@ import java.util.Optional;
@JsonSubTypes({
@JsonSubTypes.Type(value = AnonymousCredentials.class, name = "anonymous"),
@JsonSubTypes.Type(value = BasicCredentials.class, name = "basic"),
@JsonSubTypes.Type(value = AzureIotHubSasCredentials.class, name = "sas"),
@JsonSubTypes.Type(value = CertPemClientCredentials.class, name = "cert.PEM")})
public interface MqttClientCredentials {

10
rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js

File diff suppressed because one or more lines are too long

28
transport/coap/src/main/resources/tb-coap-transport.yml

@ -14,8 +14,16 @@
# limitations under the License.
#
spring.main.web-environment: false
spring.main.web-application-type: none
# If you enabled process metrics you should also enable 'web-environment'.
spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}"
# If you enabled process metrics you should set 'web-application-type' to 'servlet' value.
spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}"
server:
# Server bind address (has no effect if web-environment is disabled).
address: "${HTTP_BIND_ADDRESS:0.0.0.0}"
# Server bind port (has no effect if web-environment is disabled).
port: "${HTTP_BIND_PORT:8083}"
# Zookeeper connection parameters. Used for service discovery.
zk:
@ -205,10 +213,22 @@ queue:
transport:
# For high priority notifications that require minimum latency and processing time
notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}"
poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}"
poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}"
service:
type: "${TB_SERVICE_TYPE:tb-transport}"
# Unique id for this service (autogenerated if empty)
id: "${TB_SERVICE_ID:}"
tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id.
tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id.
metrics:
# Enable/disable actuator metrics.
enabled: "${METRICS_ENABLED:false}"
management:
endpoints:
web:
exposure:
# Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics).
include: '${METRICS_ENDPOINTS_EXPOSE:info}'

16
transport/http/src/main/resources/tb-http-transport.yml

@ -206,10 +206,22 @@ queue:
transport:
# For high priority notifications that require minimum latency and processing time
notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}"
poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}"
poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}"
service:
type: "${TB_SERVICE_TYPE:tb-transport}"
# Unique id for this service (autogenerated if empty)
id: "${TB_SERVICE_ID:}"
tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id.
tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id.
metrics:
# Enable/disable actuator metrics.
enabled: "${METRICS_ENABLED:false}"
management:
endpoints:
web:
exposure:
# Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics).
include: '${METRICS_ENDPOINTS_EXPOSE:info}'

27
transport/mqtt/src/main/resources/tb-mqtt-transport.yml

@ -14,8 +14,16 @@
# limitations under the License.
#
spring.main.web-environment: false
spring.main.web-application-type: none
# If you enabled process metrics you should also enable 'web-environment'.
spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}"
# If you enabled process metrics you should set 'web-application-type' to 'servlet' value.
spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}"
server:
# Server bind address (has no effect if web-environment is disabled).
address: "${HTTP_BIND_ADDRESS:0.0.0.0}"
# Server bind port (has no effect if web-environment is disabled).
port: "${HTTP_BIND_PORT:8083}"
# Zookeeper connection parameters. Used for service discovery.
zk:
@ -226,10 +234,21 @@ queue:
transport:
# For high priority notifications that require minimum latency and processing time
notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}"
poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}"
poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}"
service:
type: "${TB_SERVICE_TYPE:tb-transport}"
# Unique id for this service (autogenerated if empty)
id: "${TB_SERVICE_ID:}"
tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id.
tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id.
metrics:
# Enable/disable actuator metrics.
enabled: "${METRICS_ENABLED:false}"
management:
endpoints:
web:
exposure:
# Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics).
include: '${METRICS_ENDPOINTS_EXPOSE:info}'

2
ui/src/app/api/time.service.js

@ -25,7 +25,7 @@ const DAY = 24 * HOUR;
const MIN_INTERVAL = SECOND;
const MAX_INTERVAL = 365 * 20 * DAY;
const MIN_LIMIT = 10;
const MIN_LIMIT = 7;
//const AVG_LIMIT = 200;
//const MAX_LIMIT = 500;

Loading…
Cancel
Save