Browse Source

Merge pull request #3688 from thingsboard/feature/usage-records

Feature/usage records
pull/3694/head
Andrew Shvayka 6 years ago
committed by GitHub
parent
commit
a8ea887eac
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 11
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  2. 9
      application/src/main/java/org/thingsboard/server/actors/app/AppActor.java
  3. 16
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
  4. 7
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java
  5. 6
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java
  6. 51
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java
  7. 6
      application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java
  8. 63
      application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java
  9. 3
      application/src/main/java/org/thingsboard/server/controller/DeviceController.java
  10. 4
      application/src/main/java/org/thingsboard/server/controller/RuleChainController.java
  11. 2
      application/src/main/java/org/thingsboard/server/controller/TenantController.java
  12. 29
      application/src/main/java/org/thingsboard/server/service/apiusage/ApiFeature.java
  13. 357
      application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java
  14. 38
      application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java
  15. 201
      application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java
  16. 11
      application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java
  17. 56
      application/src/main/java/org/thingsboard/server/service/profile/DefaultTbDeviceProfileCache.java
  18. 4
      application/src/main/java/org/thingsboard/server/service/profile/TbDeviceProfileCache.java
  19. 25
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java
  20. 111
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  21. 48
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
  22. 3
      application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java
  23. 27
      application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java
  24. 64
      application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java
  25. 13
      application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java
  26. 2
      application/src/main/java/org/thingsboard/server/service/script/JsExecutorService.java
  27. 5
      application/src/main/java/org/thingsboard/server/service/script/JsInvokeService.java
  28. 6
      application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java
  29. 6
      application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java
  30. 11
      application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java
  31. 30
      application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java
  32. 3
      application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java
  33. 1
      application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java
  34. 8
      application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java
  35. 6
      application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java
  36. 109
      application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java
  37. 46
      application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java
  38. 2
      application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java
  39. 20
      application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java
  40. 1
      application/src/main/resources/thingsboard.yml
  41. 7
      application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java
  42. 2
      application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java
  43. 250
      application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java
  44. 57
      application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java
  45. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateService.java
  46. 4
      common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java
  47. 33
      common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java
  48. 38
      common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java
  49. 68
      common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java
  50. 2
      common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java
  51. 12
      common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java
  52. 26
      common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileType.java
  53. 3
      common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java
  54. 42
      common/data/src/main/java/org/thingsboard/server/common/data/id/ApiUsageStateId.java
  55. 2
      common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java
  56. 19
      common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java
  57. 5
      common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java
  58. 2
      common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java
  59. 28
      common/data/src/main/java/org/thingsboard/server/common/data/query/ApiUsageStateFilter.java
  60. 1
      common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java
  61. 3
      common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilterType.java
  62. 69
      common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java
  63. 43
      common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java
  64. 9
      common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java
  65. 66
      common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java
  66. 4
      common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java
  67. 48
      common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java
  68. 1
      common/message/src/main/proto/tbmsg.proto
  69. 45
      common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java
  70. 45
      common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java
  71. 12
      common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java
  72. 6
      common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java
  73. 6
      common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java
  74. 12
      common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java
  75. 7
      common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java
  76. 23
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java
  77. 23
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java
  78. 11
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java
  79. 11
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java
  80. 12
      common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java
  81. 12
      common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java
  82. 6
      common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java
  83. 6
      common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java
  84. 52
      common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java
  85. 12
      common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java
  86. 6
      common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java
  87. 6
      common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java
  88. 12
      common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java
  89. 23
      common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java
  90. 17
      common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java
  91. 36
      common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java
  92. 10
      common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java
  93. 34
      common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java
  94. 7
      common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java
  95. 35
      common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java
  96. 2
      common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java
  97. 10
      common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java
  98. 28
      common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java
  99. 26
      common/queue/src/main/java/org/thingsboard/server/queue/provider/TbUsageStatsClientQueueFactory.java
  100. 61
      common/queue/src/main/java/org/thingsboard/server/queue/scheduler/DefaultSchedulerComponent.java

11
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -32,7 +32,6 @@ import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache;
import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.service.ActorService;
import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.actors.tenant.DebugTbRateLimits;
import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.DataConstants;
@ -65,6 +64,8 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.DbCallbackExecutorService;
@ -106,6 +107,14 @@ public class ActorSystemContext {
return debugPerTenantLimits; return debugPerTenantLimits;
} }
@Autowired
@Getter
private TbApiUsageStateService apiUsageStateService;
@Autowired
@Getter
private TbApiUsageClient apiUsageClient;
@Autowired @Autowired
@Getter @Getter
@Setter @Setter

9
application/src/main/java/org/thingsboard/server/actors/app/AppActor.java

@ -40,8 +40,6 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg;
import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.service.profile.TbTenantProfileCache; import org.thingsboard.server.service.profile.TbTenantProfileCache;
import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper;
@ -150,15 +148,12 @@ public class AppActor extends ContextAwareActor {
private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) {
TbActorRef target = null; TbActorRef target = null;
if (TenantId.SYS_TENANT_ID.equals(msg.getTenantId())) { if (TenantId.SYS_TENANT_ID.equals(msg.getTenantId())) {
if (msg.getEntityId().getEntityType() == EntityType.TENANT_PROFILE) { if (!EntityType.TENANT_PROFILE.equals(msg.getEntityId().getEntityType())) {
tenantProfileCache.evict(new TenantProfileId(msg.getEntityId().getId()));
} else {
log.warn("Message has system tenant id: {}", msg); log.warn("Message has system tenant id: {}", msg);
} }
} else { } else {
if (msg.getEntityId().getEntityType() == EntityType.TENANT) { if (EntityType.TENANT.equals(msg.getEntityId().getEntityType())) {
TenantId tenantId = new TenantId(msg.getEntityId().getId()); TenantId tenantId = new TenantId(msg.getEntityId().getId());
tenantProfileCache.evict(tenantId);
if (msg.getEvent() == ComponentLifecycleEvent.DELETED) { if (msg.getEvent() == ComponentLifecycleEvent.DELETED) {
log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg);
deletedTenants.add(tenantId); deletedTenants.add(tenantId);

16
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java

@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.Alarm;
import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.RuleNodeId;
@ -73,6 +74,7 @@ import org.thingsboard.server.service.script.RuleNodeJsScriptEngine;
import java.util.Collections; import java.util.Collections;
import java.util.Set; import java.util.Set;
import java.util.function.BiConsumer;
import java.util.function.Consumer; import java.util.function.Consumer;
/** /**
@ -312,7 +314,7 @@ class DefaultTbContext implements TbContext {
@Override @Override
public ScriptEngine createJsScriptEngine(String script, String... argNames) { public ScriptEngine createJsScriptEngine(String script, String... argNames) {
return new RuleNodeJsScriptEngine(mainCtx.getJsSandbox(), nodeCtx.getSelf().getId(), script, argNames); return new RuleNodeJsScriptEngine(getTenantId(), mainCtx.getJsSandbox(), nodeCtx.getSelf().getId(), script, argNames);
} }
@Override @Override
@ -479,8 +481,16 @@ class DefaultTbContext implements TbContext {
} }
@Override @Override
public void addProfileListener(Consumer<DeviceProfile> listener) { public void removeRuleNodeStateForEntity(EntityId entityId) {
mainCtx.getDeviceProfileCache().addListener(getTenantId(), getSelfId(), listener); if (log.isDebugEnabled()) {
log.debug("[{}][{}][{}] Remove Rule Node State for entity.", getTenantId(), getSelfId(), entityId);
}
mainCtx.getRuleNodeStateService().removeByRuleNodeIdAndEntityId(getTenantId(), getSelfId(), entityId);
}
@Override
public void addDeviceProfileListeners(Consumer<DeviceProfile> profileListener, BiConsumer<DeviceId, DeviceProfile> deviceListener) {
mainCtx.getDeviceProfileCache().addListener(getTenantId(), getSelfId(), profileListener, deviceListener);
} }
@Override @Override

7
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java

@ -23,6 +23,7 @@ import org.thingsboard.server.actors.TbActorRef;
import org.thingsboard.server.actors.TbEntityActorId; import org.thingsboard.server.actors.TbEntityActorId;
import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.service.DefaultActorService;
import org.thingsboard.server.actors.shared.ComponentMsgProcessor; import org.thingsboard.server.actors.shared.ComponentMsgProcessor;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleChainId;
@ -46,6 +47,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueCallback;
import org.thingsboard.server.queue.common.MultipleTbQueueTbMsgCallbackWrapper; import org.thingsboard.server.queue.common.MultipleTbQueueTbMsgCallbackWrapper;
import org.thingsboard.server.queue.common.TbQueueTbMsgCallbackWrapper; import org.thingsboard.server.queue.common.TbQueueTbMsgCallbackWrapper;
import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.queue.TbClusterService;
import java.util.ArrayList; import java.util.ArrayList;
@ -68,15 +70,16 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
private final Map<RuleNodeId, List<RuleNodeRelation>> nodeRoutes; private final Map<RuleNodeId, List<RuleNodeRelation>> nodeRoutes;
private final RuleChainService service; private final RuleChainService service;
private final TbClusterService clusterService; private final TbClusterService clusterService;
private final TbApiUsageClient apiUsageClient;
private String ruleChainName; private String ruleChainName;
private RuleNodeId firstId; private RuleNodeId firstId;
private RuleNodeCtx firstNode; private RuleNodeCtx firstNode;
private boolean started; private boolean started;
RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext, TbActorRef parent, TbActorRef self) {
, TbActorRef parent, TbActorRef self) {
super(systemContext, tenantId, ruleChain.getId()); super(systemContext, tenantId, ruleChain.getId());
this.apiUsageClient = systemContext.getApiUsageClient();
this.ruleChainName = ruleChain.getName(); this.ruleChainName = ruleChain.getName();
this.parent = parent; this.parent = parent;
this.self = self; this.self = self;

6
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java

@ -64,6 +64,12 @@ public abstract class RuleChainManagerActor extends ContextAwareActor {
} }
} }
protected void destroyRuleChains() {
for (RuleChain ruleChain : new PageDataIterable<>(link -> ruleChainService.findTenantRuleChains(tenantId, link), ContextAwareActor.ENTITY_PACK_LIMIT)) {
ctx.stop(new TbEntityActorId(ruleChain.getId()));
}
}
protected void visit(RuleChain entity, TbActorRef actorRef) { protected void visit(RuleChain entity, TbActorRef actorRef) {
if (entity != null && entity.isRoot()) { if (entity != null && entity.isRoot()) {
rootChain = entity; rootChain = entity;

51
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java

@ -21,13 +21,18 @@ import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorCtx;
import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.TbActorRef;
import org.thingsboard.server.actors.shared.ComponentMsgProcessor; import org.thingsboard.server.actors.shared.ComponentMsgProcessor;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; import org.thingsboard.server.common.data.plugin.ComponentLifecycleState;
import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.PartitionChangeMsg;
import org.thingsboard.server.common.msg.queue.RuleNodeException; import org.thingsboard.server.common.msg.queue.RuleNodeException;
import org.thingsboard.server.common.msg.queue.RuleNodeInfo; import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
/** /**
* @author Andrew Shvayka * @author Andrew Shvayka
@ -36,6 +41,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
private final String ruleChainName; private final String ruleChainName;
private final TbActorRef self; private final TbActorRef self;
private final TbApiUsageClient apiUsageClient;
private RuleNode ruleNode; private RuleNode ruleNode;
private TbNode tbNode; private TbNode tbNode;
private DefaultTbContext defaultCtx; private DefaultTbContext defaultCtx;
@ -44,6 +50,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
RuleNodeActorMessageProcessor(TenantId tenantId, String ruleChainName, RuleNodeId ruleNodeId, ActorSystemContext systemContext RuleNodeActorMessageProcessor(TenantId tenantId, String ruleChainName, RuleNodeId ruleNodeId, ActorSystemContext systemContext
, TbActorRef parent, TbActorRef self) { , TbActorRef parent, TbActorRef self) {
super(systemContext, tenantId, ruleNodeId); super(systemContext, tenantId, ruleNodeId);
this.apiUsageClient = systemContext.getApiUsageClient();
this.ruleChainName = ruleChainName; this.ruleChainName = ruleChainName;
this.self = self; this.self = self;
this.ruleNode = systemContext.getRuleChainService().findRuleNodeById(tenantId, entityId); this.ruleNode = systemContext.getRuleChainService().findRuleNodeById(tenantId, entityId);
@ -92,26 +99,42 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
public void onRuleToSelfMsg(RuleNodeToSelfMsg msg) throws Exception { public void onRuleToSelfMsg(RuleNodeToSelfMsg msg) throws Exception {
checkActive(msg.getMsg()); checkActive(msg.getMsg());
if (ruleNode.isDebugMode()) { TbMsg tbMsg = msg.getMsg();
systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), "Self"); int ruleNodeCount = tbMsg.getAndIncrementRuleNodeCounter();
} int maxRuleNodeExecutionsPerMessage = getTenantProfileConfiguration().getMaxRuleNodeExecsPerMessage();
try { if (maxRuleNodeExecutionsPerMessage == 0 || ruleNodeCount < maxRuleNodeExecutionsPerMessage) {
tbNode.onMsg(defaultCtx, msg.getMsg()); apiUsageClient.report(tenantId, ApiUsageRecordKey.RE_EXEC_COUNT);
} catch (Exception e) { if (ruleNode.isDebugMode()) {
defaultCtx.tellFailure(msg.getMsg(), e); systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), "Self");
}
try {
tbNode.onMsg(defaultCtx, msg.getMsg());
} catch (Exception e) {
defaultCtx.tellFailure(msg.getMsg(), e);
}
} else {
tbMsg.getCallback().onFailure(new RuleNodeException("Message is processed by more then " + maxRuleNodeExecutionsPerMessage + " rule nodes!", ruleChainName, ruleNode));
} }
} }
void onRuleChainToRuleNodeMsg(RuleChainToRuleNodeMsg msg) throws Exception { void onRuleChainToRuleNodeMsg(RuleChainToRuleNodeMsg msg) throws Exception {
msg.getMsg().getCallback().onProcessingStart(info); msg.getMsg().getCallback().onProcessingStart(info);
checkActive(msg.getMsg()); checkActive(msg.getMsg());
if (ruleNode.isDebugMode()) { TbMsg tbMsg = msg.getMsg();
systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), msg.getFromRelationType()); int ruleNodeCount = tbMsg.getAndIncrementRuleNodeCounter();
} int maxRuleNodeExecutionsPerMessage = getTenantProfileConfiguration().getMaxRuleNodeExecsPerMessage();
try { if (maxRuleNodeExecutionsPerMessage == 0 || ruleNodeCount < maxRuleNodeExecutionsPerMessage) {
tbNode.onMsg(msg.getCtx(), msg.getMsg()); apiUsageClient.report(tenantId, ApiUsageRecordKey.RE_EXEC_COUNT);
} catch (Exception e) { if (ruleNode.isDebugMode()) {
msg.getCtx().tellFailure(msg.getMsg(), e); systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), msg.getFromRelationType());
}
try {
tbNode.onMsg(msg.getCtx(), msg.getMsg());
} catch (Exception e) {
msg.getCtx().tellFailure(msg.getMsg(), e);
}
} else {
tbMsg.getCallback().onFailure(new RuleNodeException("Message is processed by more then " + maxRuleNodeExecutionsPerMessage + " rule nodes!", ruleChainName, ruleNode));
} }
} }

6
application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java

@ -19,9 +19,11 @@ import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorCtx;
import org.thingsboard.server.actors.stats.StatsPersistTick; import org.thingsboard.server.actors.stats.StatsPersistTick;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; import org.thingsboard.server.common.data.plugin.ComponentLifecycleState;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.PartitionChangeMsg;
import org.thingsboard.server.common.msg.queue.RuleNodeException; import org.thingsboard.server.common.msg.queue.RuleNodeException;
@ -39,6 +41,10 @@ public abstract class ComponentMsgProcessor<T extends EntityId> extends Abstract
this.entityId = id; this.entityId = id;
} }
protected TenantProfileConfiguration getTenantProfileConfiguration() {
return systemContext.getTenantProfileCache().get(tenantId).getProfileData().getConfiguration();
}
public abstract String getComponentName(); public abstract String getComponentName();
public abstract void start(TbActorCtx context) throws Exception; public abstract void start(TbActorCtx context) throws Exception;

63
application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java

@ -29,6 +29,7 @@ import org.thingsboard.server.actors.device.DeviceActorCreator;
import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor;
import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.ContextBasedCreator;
import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.service.DefaultActorService;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.TenantProfile;
@ -57,6 +58,7 @@ public class TenantActor extends RuleChainManagerActor {
private boolean isRuleEngineForCurrentTenant; private boolean isRuleEngineForCurrentTenant;
private boolean isCore; private boolean isCore;
private ApiUsageState apiUsageState;
private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { private TenantActor(ActorSystemContext systemContext, TenantId tenantId) {
super(systemContext, tenantId); super(systemContext, tenantId);
@ -74,19 +76,24 @@ public class TenantActor extends RuleChainManagerActor {
cantFindTenant = true; cantFindTenant = true;
log.info("[{}] Started tenant actor for missing tenant.", tenantId); log.info("[{}] Started tenant actor for missing tenant.", tenantId);
} else { } else {
apiUsageState = new ApiUsageState(systemContext.getApiUsageStateService().getApiUsageState(tenant.getId()));
// This Service may be started for specific tenant only. // This Service may be started for specific tenant only.
Optional<TenantId> isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); Optional<TenantId> isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant();
TenantProfile tenantProfile = systemContext.getTenantProfileCache().get(tenant.getTenantProfileId()); TenantProfile tenantProfile = systemContext.getTenantProfileCache().get(tenant.getTenantProfileId());
isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE);
isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE);
isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE);
if (isRuleEngineForCurrentTenant) { if (isRuleEngineForCurrentTenant) {
try { try {
if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenantProfile.isIsolatedTbRuleEngine())) { if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenantProfile.isIsolatedTbRuleEngine())) {
log.info("[{}] Going to init rule chains", tenantId); if (apiUsageState.isReExecEnabled()) {
initRuleChains(); log.info("[{}] Going to init rule chains", tenantId);
initRuleChains();
} else {
log.info("[{}] Skip init of the rule chains due to API limits", tenantId);
}
} else { } else {
isRuleEngineForCurrentTenant = false; isRuleEngineForCurrentTenant = false;
} }
@ -98,8 +105,6 @@ public class TenantActor extends RuleChainManagerActor {
} }
} catch (Exception e) { } catch (Exception e) {
log.warn("[{}] Unknown failure", tenantId, e); log.warn("[{}] Unknown failure", tenantId, e);
// TODO: throw this in 3.1?
// throw new TbActorException("Failed to init actor", e);
} }
} }
@ -115,7 +120,7 @@ public class TenantActor extends RuleChainManagerActor {
if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) { if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) {
QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg; QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg;
queueMsg.getTbMsg().getCallback().onSuccess(); queueMsg.getTbMsg().getCallback().onSuccess();
} else if (msg.getMsgType().equals(MsgType.TRANSPORT_TO_DEVICE_ACTOR_MSG)){ } else if (msg.getMsgType().equals(MsgType.TRANSPORT_TO_DEVICE_ACTOR_MSG)) {
TransportToDeviceActorMsgWrapper transportMsg = (TransportToDeviceActorMsgWrapper) msg; TransportToDeviceActorMsgWrapper transportMsg = (TransportToDeviceActorMsgWrapper) msg;
transportMsg.getCallback().onSuccess(); transportMsg.getCallback().onSuccess();
} }
@ -173,26 +178,33 @@ public class TenantActor extends RuleChainManagerActor {
return; return;
} }
TbMsg tbMsg = msg.getTbMsg(); TbMsg tbMsg = msg.getTbMsg();
if (tbMsg.getRuleChainId() == null) { if (apiUsageState.isReExecEnabled()) {
if (getRootChainActor() != null) { if (tbMsg.getRuleChainId() == null) {
getRootChainActor().tell(msg); if (getRootChainActor() != null) {
getRootChainActor().tell(msg);
} else {
tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!"));
log.info("[{}] No Root Chain: {}", tenantId, msg);
}
} else { } else {
tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!")); try {
log.info("[{}] No Root Chain: {}", tenantId, msg); ctx.tell(new TbEntityActorId(tbMsg.getRuleChainId()), msg);
} catch (TbActorNotRegisteredException ex) {
log.trace("Received message for non-existing rule chain: [{}]", tbMsg.getRuleChainId());
//TODO: 3.1 Log it to dead letters queue;
tbMsg.getCallback().onSuccess();
}
} }
} else { } else {
try { log.trace("[{}] Ack message because Rule Engine is disabled", tenantId);
ctx.tell(new TbEntityActorId(tbMsg.getRuleChainId()), msg); tbMsg.getCallback().onSuccess();
} catch (TbActorNotRegisteredException ex) {
log.trace("Received message for non-existing rule chain: [{}]", tbMsg.getRuleChainId());
//TODO: 3.1 Log it to dead letters queue;
tbMsg.getCallback().onSuccess();
}
} }
} }
private void onRuleChainMsg(RuleChainAwareMsg msg) { private void onRuleChainMsg(RuleChainAwareMsg msg) {
getOrCreateActor(msg.getRuleChainId()).tell(msg); if (apiUsageState.isReExecEnabled()) {
getOrCreateActor(msg.getRuleChainId()).tell(msg);
}
} }
private void onToDeviceActorMsg(DeviceAwareMsg msg, boolean priority) { private void onToDeviceActorMsg(DeviceAwareMsg msg, boolean priority) {
@ -208,6 +220,17 @@ public class TenantActor extends RuleChainManagerActor {
} }
private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) {
if (msg.getEntityId().getEntityType().equals(EntityType.API_USAGE_STATE)) {
ApiUsageState old = apiUsageState;
apiUsageState = new ApiUsageState(systemContext.getApiUsageStateService().getApiUsageState(tenantId));
if (old.isReExecEnabled() && !apiUsageState.isReExecEnabled()) {
log.info("[{}] Received API state update. Going to DISABLE Rule Engine execution.", tenantId);
destroyRuleChains();
} else if (!old.isReExecEnabled() && apiUsageState.isReExecEnabled()) {
log.info("[{}] Received API state update. Going to ENABLE Rule Engine execution.", tenantId);
initRuleChains();
}
}
if (isRuleEngineForCurrentTenant) { if (isRuleEngineForCurrentTenant) {
TbActorRef target = getEntityActorRef(msg.getEntityId()); TbActorRef target = getEntityActorRef(msg.getEntityId());
if (target != null) { if (target != null) {

3
application/src/main/java/org/thingsboard/server/controller/DeviceController.java

@ -51,6 +51,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgDataType;
@ -119,6 +120,8 @@ public class DeviceController extends BaseController {
tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(),
savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null); savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null);
tbClusterService.onEntityStateChange(savedDevice.getTenantId(), savedDevice.getId(),
device.getId() == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
logEntityAction(savedDevice.getId(), savedDevice, logEntityAction(savedDevice.getId(), savedDevice,
savedDevice.getCustomerId(), savedDevice.getCustomerId(),

4
application/src/main/java/org/thingsboard/server/controller/RuleChainController.java

@ -160,7 +160,7 @@ public class RuleChainController extends BaseController {
public RuleChain saveRuleChain(@RequestBody DefaultRuleChainCreateRequest request) throws ThingsboardException { public RuleChain saveRuleChain(@RequestBody DefaultRuleChainCreateRequest request) throws ThingsboardException {
try { try {
checkNotNull(request); checkNotNull(request);
checkNotNull(request.getName()); checkParameter(request.getName(), "name");
RuleChain savedRuleChain = installScripts.createDefaultRuleChain(getCurrentUser().getTenantId(), request.getName()); RuleChain savedRuleChain = installScripts.createDefaultRuleChain(getCurrentUser().getTenantId(), request.getName());
@ -347,7 +347,7 @@ public class RuleChainController extends BaseController {
String errorText = ""; String errorText = "";
ScriptEngine engine = null; ScriptEngine engine = null;
try { try {
engine = new RuleNodeJsScriptEngine(jsInvokeService, getCurrentUser().getId(), script, argNames); engine = new RuleNodeJsScriptEngine(getTenantId(), jsInvokeService, getCurrentUser().getId(), script, argNames);
TbMsg inMsg = TbMsg.newMsg(msgType, null, new TbMsgMetaData(metadata), TbMsgDataType.JSON, data); TbMsg inMsg = TbMsg.newMsg(msgType, null, new TbMsgMetaData(metadata), TbMsgDataType.JSON, data);
switch (scriptType) { switch (scriptType) {
case "update": case "update":

2
application/src/main/java/org/thingsboard/server/controller/TenantController.java

@ -93,6 +93,8 @@ public class TenantController extends BaseController {
} }
tenantProfileCache.evict(tenant.getId()); tenantProfileCache.evict(tenant.getId());
tbClusterService.onTenantChange(tenant, null); tbClusterService.onTenantChange(tenant, null);
tbClusterService.onEntityStateChange(tenant.getId(), tenant.getId(),
newTenant ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
return tenant; return tenant;
} catch (Exception e) { } catch (Exception e) {
throw handleException(e); throw handleException(e);

29
application/src/main/java/org/thingsboard/server/service/apiusage/ApiFeature.java

@ -0,0 +1,29 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.apiusage;
import lombok.Getter;
public enum ApiFeature {
TRANSPORT("transportApiState"), DB("dbApiState"), RE("ruleEngineApiState"), JS("jsExecutionApiState");
@Getter
private final String apiStateKey;
ApiFeature(String apiStateKey) {
this.apiStateKey = apiStateKey;
}
}

357
application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java

@ -0,0 +1,357 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.apiusage;
import com.google.common.util.concurrent.FutureCallback;
import lombok.extern.slf4j.Slf4j;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.data.util.Pair;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.ApiUsageStateId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.TenantProfileId;
import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
import org.thingsboard.server.common.data.kv.BooleanDataEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.data.page.PageDataIterable;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.tools.SchedulerUtils;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.scheduler.SchedulerComponent;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.profile.TbTenantProfileCache;
import org.thingsboard.server.service.queue.TbClusterService;
import org.thingsboard.server.service.telemetry.InternalTelemetryService;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@Slf4j
@TbCoreComponent
@Service
public class DefaultTbApiUsageStateService implements TbApiUsageStateService {
public static final String HOURLY = "Hourly";
public static final FutureCallback<Integer> VOID_CALLBACK = new FutureCallback<Integer>() {
@Override
public void onSuccess(@Nullable Integer result) {
}
@Override
public void onFailure(Throwable t) {
}
};
private final TbClusterService clusterService;
private final PartitionService partitionService;
private final TenantService tenantService;
private final TimeseriesService tsService;
private final ApiUsageStateService apiUsageStateService;
private final SchedulerComponent scheduler;
private final TbTenantProfileCache tenantProfileCache;
@Lazy
@Autowired
private InternalTelemetryService tsWsService;
// Tenants that should be processed on this server
private final Map<TenantId, TenantApiUsageState> myTenantStates = new ConcurrentHashMap<>();
// Tenants that should be processed on other servers
private final Map<TenantId, ApiUsageState> otherTenantStates = new ConcurrentHashMap<>();
@Value("${usage.stats.report.enabled:true}")
private boolean enabled;
@Value("${usage.stats.check.cycle:60000}")
private long nextCycleCheckInterval;
private final Lock updateLock = new ReentrantLock();
public DefaultTbApiUsageStateService(TbClusterService clusterService,
PartitionService partitionService,
TenantService tenantService,
TimeseriesService tsService,
ApiUsageStateService apiUsageStateService,
SchedulerComponent scheduler,
TbTenantProfileCache tenantProfileCache) {
this.clusterService = clusterService;
this.partitionService = partitionService;
this.tenantService = tenantService;
this.tsService = tsService;
this.apiUsageStateService = apiUsageStateService;
this.scheduler = scheduler;
this.tenantProfileCache = tenantProfileCache;
}
@PostConstruct
public void init() {
if (enabled) {
log.info("Starting api usage service.");
initStatesFromDataBase();
scheduler.scheduleAtFixedRate(this::checkStartOfNextCycle, nextCycleCheckInterval, nextCycleCheckInterval, TimeUnit.MILLISECONDS);
}
}
@Override
public void process(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback) {
ToUsageStatsServiceMsg statsMsg = msg.getValue();
TenantId tenantId = new TenantId(new UUID(statsMsg.getTenantIdMSB(), statsMsg.getTenantIdLSB()));
TenantApiUsageState tenantState;
List<TsKvEntry> updatedEntries;
Map<ApiFeature, Boolean> result = new HashMap<>();
updateLock.lock();
try {
tenantState = getOrFetchState(tenantId);
long ts = tenantState.getCurrentCycleTs();
long hourTs = tenantState.getCurrentHourTs();
long newHourTs = SchedulerUtils.getStartOfCurrentHour();
if (newHourTs != hourTs) {
tenantState.setHour(newHourTs);
}
updatedEntries = new ArrayList<>(ApiUsageRecordKey.values().length);
for (UsageStatsKVProto kvProto : statsMsg.getValuesList()) {
ApiUsageRecordKey recordKey = ApiUsageRecordKey.valueOf(kvProto.getKey());
long newValue = tenantState.add(recordKey, kvProto.getValue());
updatedEntries.add(new BasicTsKvEntry(ts, new LongDataEntry(recordKey.getApiCountKey(), newValue)));
long newHourlyValue = tenantState.addToHourly(recordKey, kvProto.getValue());
updatedEntries.add(new BasicTsKvEntry(hourTs, new LongDataEntry(recordKey.getApiCountKey() + HOURLY, newHourlyValue)));
Pair<ApiFeature, Boolean> update = tenantState.checkStateUpdatedDueToThreshold(recordKey);
if (update != null) {
result.put(update.getFirst(), update.getSecond());
}
}
} finally {
updateLock.unlock();
}
tsWsService.saveAndNotifyInternal(tenantId, tenantState.getApiUsageState().getId(), updatedEntries, VOID_CALLBACK);
if (!result.isEmpty()) {
persistAndNotify(tenantState, result);
}
callback.onSuccess();
}
@Override
public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
if (partitionChangeEvent.getServiceType().equals(ServiceType.TB_CORE)) {
myTenantStates.entrySet().removeIf(entry -> !partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition());
otherTenantStates.entrySet().removeIf(entry -> partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition());
initStatesFromDataBase();
}
}
@Override
public ApiUsageState getApiUsageState(TenantId tenantId) {
TenantApiUsageState tenantState = myTenantStates.get(tenantId);
if (tenantState != null) {
return tenantState.getApiUsageState();
} else {
ApiUsageState state = otherTenantStates.get(tenantId);
if (state != null) {
return state;
} else {
if (partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) {
return getOrFetchState(tenantId).getApiUsageState();
} else {
updateLock.lock();
try {
state = otherTenantStates.get(tenantId);
if (state == null) {
state = apiUsageStateService.findTenantApiUsageState(tenantId);
otherTenantStates.put(tenantId, state);
}
} finally {
updateLock.unlock();
}
return state;
}
}
}
}
@Override
public void onApiUsageStateUpdate(TenantId tenantId) {
otherTenantStates.remove(tenantId);
}
@Override
public void onTenantProfileUpdate(TenantProfileId tenantProfileId) {
TenantProfile tenantProfile = tenantProfileCache.get(tenantProfileId);
updateLock.lock();
try {
myTenantStates.values().forEach(state -> {
if (tenantProfile.getId().equals(state.getTenantProfileId())) {
updateTenantState(state, tenantProfile);
}
});
} finally {
updateLock.unlock();
}
}
@Override
public void onTenantUpdate(TenantId tenantId) {
TenantProfile tenantProfile = tenantProfileCache.get(tenantId);
updateLock.lock();
try {
TenantApiUsageState state = myTenantStates.get(tenantId);
if (state != null && !state.getTenantProfileId().equals(tenantProfile.getId())) {
updateTenantState(state, tenantProfile);
}
} finally {
updateLock.unlock();
}
}
private void updateTenantState(TenantApiUsageState state, TenantProfile tenantProfile) {
TenantProfileData oldProfileData = state.getTenantProfileData();
state.setTenantProfileId(tenantProfile.getId());
state.setTenantProfileData(tenantProfile.getProfileData());
Map<ApiFeature, Boolean> result = state.checkStateUpdatedDueToThresholds();
if (!result.isEmpty()) {
persistAndNotify(state, result);
}
updateProfileThresholds(state.getTenantId(), state.getApiUsageState().getId(),
oldProfileData.getConfiguration(), tenantProfile.getProfileData().getConfiguration());
}
private void updateProfileThresholds(TenantId tenantId, ApiUsageStateId id,
TenantProfileConfiguration oldData, TenantProfileConfiguration newData) {
long ts = System.currentTimeMillis();
List<TsKvEntry> profileThresholds = new ArrayList<>();
for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
long newProfileThreshold = newData.getProfileThreshold(key);
if (oldData == null || oldData.getProfileThreshold(key) != newProfileThreshold) {
log.info("[{}] Updating profile threshold [{}]:[{}]", tenantId, key, newProfileThreshold);
profileThresholds.add(new BasicTsKvEntry(ts, new LongDataEntry(key.getApiLimitKey(), newProfileThreshold)));
}
}
if (!profileThresholds.isEmpty()) {
tsWsService.saveAndNotifyInternal(tenantId, id, profileThresholds, VOID_CALLBACK);
}
}
private void persistAndNotify(TenantApiUsageState state, Map<ApiFeature, Boolean> result) {
log.info("[{}] Detected update of the API state: {}", state.getTenantId(), result);
apiUsageStateService.update(state.getApiUsageState());
clusterService.onApiStateChange(state.getApiUsageState(), null);
long ts = System.currentTimeMillis();
List<TsKvEntry> stateTelemetry = new ArrayList<>();
result.forEach(((apiFeature, aState) -> stateTelemetry.add(new BasicTsKvEntry(ts, new BooleanDataEntry(apiFeature.getApiStateKey(), aState)))));
tsWsService.saveAndNotifyInternal(state.getTenantId(), state.getApiUsageState().getId(), stateTelemetry, VOID_CALLBACK);
}
private void checkStartOfNextCycle() {
updateLock.lock();
try {
long now = System.currentTimeMillis();
myTenantStates.values().forEach(state -> {
if ((state.getNextCycleTs() > now) && (state.getNextCycleTs() - now < TimeUnit.HOURS.toMillis(1))) {
state.setCycles(state.getNextCycleTs(), SchedulerUtils.getStartOfNextNextMonth());
}
});
} finally {
updateLock.unlock();
}
}
private TenantApiUsageState getOrFetchState(TenantId tenantId) {
TenantApiUsageState tenantState = myTenantStates.get(tenantId);
if (tenantState == null) {
ApiUsageState dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId);
if (dbStateEntity == null) {
try {
dbStateEntity = apiUsageStateService.createDefaultApiUsageState(tenantId);
} catch (Exception e) {
dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId);
}
}
TenantProfile tenantProfile = tenantProfileCache.get(tenantId);
tenantState = new TenantApiUsageState(tenantProfile, dbStateEntity);
try {
List<TsKvEntry> dbValues = tsService.findAllLatest(tenantId, dbStateEntity.getId()).get();
for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
boolean cycleEntryFound = false;
boolean hourlyEntryFound = false;
for (TsKvEntry tsKvEntry : dbValues) {
if (tsKvEntry.getKey().equals(key.getApiCountKey())) {
cycleEntryFound = true;
tenantState.put(key, tsKvEntry.getTs() == tenantState.getCurrentCycleTs() ? tsKvEntry.getLongValue().get() : 0L);
} else if (tsKvEntry.getKey().equals(key.getApiCountKey() + HOURLY)) {
hourlyEntryFound = true;
tenantState.putHourly(key, tsKvEntry.getTs() == tenantState.getCurrentHourTs() ? tsKvEntry.getLongValue().get() : 0L);
}
if (cycleEntryFound && hourlyEntryFound) {
break;
}
}
}
log.debug("[{}] Initialized state: {}", tenantId, dbStateEntity);
myTenantStates.put(tenantId, tenantState);
} catch (InterruptedException | ExecutionException e) {
log.warn("[{}] Failed to fetch api usage state from db.", tenantId, e);
}
}
return tenantState;
}
private void initStatesFromDataBase() {
try {
PageDataIterable<Tenant> tenantIterator = new PageDataIterable<>(tenantService::findTenants, 1024);
for (Tenant tenant : tenantIterator) {
if (!myTenantStates.containsKey(tenant.getId()) && partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), tenant.getId()).isMyPartition()) {
updateLock.lock();
try {
updateTenantState(getOrFetchState(tenant.getId()), tenantProfileCache.get(tenant.getTenantProfileId()));
} catch (Exception e) {
log.warn("[{}] Failed to initialize tenant API state", tenant.getId(), e);
} finally {
updateLock.unlock();
}
}
}
log.info("Api usage service started.");
} catch (Exception e) {
log.warn("Unknown failure", e);
}
}
}

38
application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java

@ -0,0 +1,38 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.apiusage;
import org.springframework.context.ApplicationListener;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.TenantProfileId;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
public interface TbApiUsageStateService extends ApplicationListener<PartitionChangeEvent> {
void process(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback);
ApiUsageState getApiUsageState(TenantId tenantId);
void onTenantProfileUpdate(TenantProfileId tenantProfileId);
void onTenantUpdate(TenantId tenantId);
void onApiUsageStateUpdate(TenantId tenantId);
}

201
application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java

@ -0,0 +1,201 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.apiusage;
import lombok.Getter;
import lombok.Setter;
import org.springframework.data.util.Pair;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.TenantProfileId;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import org.thingsboard.server.common.msg.tools.SchedulerUtils;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public class TenantApiUsageState {
private final Map<ApiUsageRecordKey, Long> currentCycleValues = new ConcurrentHashMap<>();
private final Map<ApiUsageRecordKey, Long> currentHourValues = new ConcurrentHashMap<>();
@Getter
@Setter
private TenantProfileId tenantProfileId;
@Getter
@Setter
private TenantProfileData tenantProfileData;
@Getter
private final ApiUsageState apiUsageState;
@Getter
private volatile long currentCycleTs;
@Getter
private volatile long nextCycleTs;
@Getter
private volatile long currentHourTs;
public TenantApiUsageState(TenantProfile tenantProfile, ApiUsageState apiUsageState) {
this.tenantProfileId = tenantProfile.getId();
this.tenantProfileData = tenantProfile.getProfileData();
this.apiUsageState = apiUsageState;
this.currentCycleTs = SchedulerUtils.getStartOfCurrentMonth();
this.nextCycleTs = SchedulerUtils.getStartOfNextMonth();
this.currentHourTs = SchedulerUtils.getStartOfCurrentHour();
}
public void put(ApiUsageRecordKey key, Long value) {
currentCycleValues.put(key, value);
}
public void putHourly(ApiUsageRecordKey key, Long value) {
currentHourValues.put(key, value);
}
public long add(ApiUsageRecordKey key, long value) {
long result = currentCycleValues.getOrDefault(key, 0L) + value;
currentCycleValues.put(key, result);
return result;
}
public long get(ApiUsageRecordKey key) {
return currentCycleValues.getOrDefault(key, 0L);
}
public long addToHourly(ApiUsageRecordKey key, long value) {
long result = currentHourValues.getOrDefault(key, 0L) + value;
currentHourValues.put(key, result);
return result;
}
public void setHour(long currentHourTs) {
this.currentHourTs = currentHourTs;
for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
currentHourValues.put(key, 0L);
}
}
public void setCycles(long currentCycleTs, long nextCycleTs) {
this.currentCycleTs = currentCycleTs;
this.nextCycleTs = nextCycleTs;
for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
currentCycleValues.put(key, 0L);
}
}
public long getProfileThreshold(ApiUsageRecordKey key) {
return tenantProfileData.getConfiguration().getProfileThreshold(key);
}
public TenantId getTenantId() {
return apiUsageState.getTenantId();
}
public boolean isTransportEnabled() {
return apiUsageState.isTransportEnabled();
}
public boolean isDbStorageEnabled() {
return apiUsageState.isDbStorageEnabled();
}
public boolean isRuleEngineEnabled() {
return apiUsageState.isReExecEnabled();
}
public boolean isJsExecEnabled() {
return apiUsageState.isJsExecEnabled();
}
public void setTransportEnabled(boolean transportEnabled) {
apiUsageState.setTransportEnabled(transportEnabled);
}
public void setDbStorageEnabled(boolean dbStorageEnabled) {
apiUsageState.setDbStorageEnabled(dbStorageEnabled);
}
public void setRuleEngineEnabled(boolean ruleEngineEnabled) {
apiUsageState.setReExecEnabled(ruleEngineEnabled);
}
public void setJsExecEnabled(boolean jsExecEnabled) {
apiUsageState.setJsExecEnabled(jsExecEnabled);
}
public boolean isFeatureEnabled(ApiUsageRecordKey recordKey) {
switch (recordKey) {
case TRANSPORT_MSG_COUNT:
case TRANSPORT_DP_COUNT:
return isTransportEnabled();
case RE_EXEC_COUNT:
return isRuleEngineEnabled();
case STORAGE_DP_COUNT:
return isDbStorageEnabled();
case JS_EXEC_COUNT:
return isJsExecEnabled();
default:
return true;
}
}
public ApiFeature setFeatureValue(ApiUsageRecordKey recordKey, boolean value) {
ApiFeature feature = null;
boolean currentValue = isFeatureEnabled(recordKey);
switch (recordKey) {
case TRANSPORT_MSG_COUNT:
case TRANSPORT_DP_COUNT:
feature = ApiFeature.TRANSPORT;
setTransportEnabled(value);
break;
case RE_EXEC_COUNT:
feature = ApiFeature.RE;
setRuleEngineEnabled(value);
break;
case STORAGE_DP_COUNT:
feature = ApiFeature.DB;
setDbStorageEnabled(value);
break;
case JS_EXEC_COUNT:
feature = ApiFeature.JS;
setJsExecEnabled(value);
break;
}
return currentValue == value ? null : feature;
}
public Map<ApiFeature, Boolean> checkStateUpdatedDueToThresholds() {
Map<ApiFeature, Boolean> result = new HashMap<>();
for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
Pair<ApiFeature, Boolean> featureUpdate = checkStateUpdatedDueToThreshold(key);
if (featureUpdate != null) {
result.put(featureUpdate.getFirst(), featureUpdate.getSecond());
}
}
return result;
}
public Pair<ApiFeature, Boolean> checkStateUpdatedDueToThreshold(ApiUsageRecordKey recordKey) {
long value = get(recordKey);
long threshold = getProfileThreshold(recordKey);
boolean featureValue = threshold == 0 || value < threshold;
ApiFeature feature = setFeatureValue(recordKey, featureValue);
return feature != null ? Pair.of(feature, featureValue) : null;
}
}

11
application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java

@ -30,7 +30,8 @@ import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.CustomerId;
@ -127,6 +128,9 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
public void createDefaultTenantProfiles() throws Exception { public void createDefaultTenantProfiles() throws Exception {
tenantProfileService.findOrCreateDefaultTenantProfile(TenantId.SYS_TENANT_ID); tenantProfileService.findOrCreateDefaultTenantProfile(TenantId.SYS_TENANT_ID);
TenantProfileData tenantProfileData = new TenantProfileData();
tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration());
TenantProfile isolatedTbCoreProfile = new TenantProfile(); TenantProfile isolatedTbCoreProfile = new TenantProfile();
isolatedTbCoreProfile.setDefault(false); isolatedTbCoreProfile.setDefault(false);
isolatedTbCoreProfile.setName("Isolated TB Core"); isolatedTbCoreProfile.setName("Isolated TB Core");
@ -134,6 +138,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
isolatedTbCoreProfile.setDescription("Isolated TB Core tenant profile"); isolatedTbCoreProfile.setDescription("Isolated TB Core tenant profile");
isolatedTbCoreProfile.setIsolatedTbCore(true); isolatedTbCoreProfile.setIsolatedTbCore(true);
isolatedTbCoreProfile.setIsolatedTbRuleEngine(false); isolatedTbCoreProfile.setIsolatedTbRuleEngine(false);
isolatedTbCoreProfile.setProfileData(tenantProfileData);
try { try {
tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreProfile); tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreProfile);
} catch (DataValidationException e) { } catch (DataValidationException e) {
@ -147,6 +152,8 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
isolatedTbRuleEngineProfile.setDescription("Isolated TB Rule Engine tenant profile"); isolatedTbRuleEngineProfile.setDescription("Isolated TB Rule Engine tenant profile");
isolatedTbRuleEngineProfile.setIsolatedTbCore(false); isolatedTbRuleEngineProfile.setIsolatedTbCore(false);
isolatedTbRuleEngineProfile.setIsolatedTbRuleEngine(true); isolatedTbRuleEngineProfile.setIsolatedTbRuleEngine(true);
isolatedTbRuleEngineProfile.setProfileData(tenantProfileData);
try { try {
tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbRuleEngineProfile); tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbRuleEngineProfile);
} catch (DataValidationException e) { } catch (DataValidationException e) {
@ -160,6 +167,8 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
isolatedTbCoreAndTbRuleEngineProfile.setDescription("Isolated TB Core and TB Rule Engine tenant profile"); isolatedTbCoreAndTbRuleEngineProfile.setDescription("Isolated TB Core and TB Rule Engine tenant profile");
isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbCore(true); isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbCore(true);
isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbRuleEngine(true); isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbRuleEngine(true);
isolatedTbCoreAndTbRuleEngineProfile.setProfileData(tenantProfileData);
try { try {
tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreAndTbRuleEngineProfile); tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreAndTbRuleEngineProfile);
} catch (DataValidationException e) { } catch (DataValidationException e) {

56
application/src/main/java/org/thingsboard/server/service/profile/DefaultTbDeviceProfileCache.java

@ -26,11 +26,11 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.device.DeviceService;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
import java.util.function.BiConsumer;
import java.util.function.Consumer; import java.util.function.Consumer;
@Service @Service
@ -43,7 +43,8 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache {
private final ConcurrentMap<DeviceProfileId, DeviceProfile> deviceProfilesMap = new ConcurrentHashMap<>(); private final ConcurrentMap<DeviceProfileId, DeviceProfile> deviceProfilesMap = new ConcurrentHashMap<>();
private final ConcurrentMap<DeviceId, DeviceProfileId> devicesMap = new ConcurrentHashMap<>(); private final ConcurrentMap<DeviceId, DeviceProfileId> devicesMap = new ConcurrentHashMap<>();
private final ConcurrentMap<TenantId, ConcurrentMap<EntityId, Consumer<DeviceProfile>>> listeners = new ConcurrentHashMap<>(); private final ConcurrentMap<TenantId, ConcurrentMap<EntityId, Consumer<DeviceProfile>>> profileListeners = new ConcurrentHashMap<>();
private final ConcurrentMap<TenantId, ConcurrentMap<EntityId, BiConsumer<DeviceId, DeviceProfile>>> deviceProfileListeners = new ConcurrentHashMap<>();
public DefaultTbDeviceProfileCache(DeviceProfileService deviceProfileService, DeviceService deviceService) { public DefaultTbDeviceProfileCache(DeviceProfileService deviceProfileService, DeviceService deviceService) {
this.deviceProfileService = deviceProfileService; this.deviceProfileService = deviceProfileService;
@ -87,33 +88,37 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache {
return get(tenantId, profileId); return get(tenantId, profileId);
} }
@Override
public void put(DeviceProfile profile) {
if (profile.getId() != null) {
deviceProfilesMap.put(profile.getId(), profile);
log.debug("[{}] pushed device profile to cache: {}", profile.getId(), profile);
notifyListeners(profile);
}
}
@Override @Override
public void evict(TenantId tenantId, DeviceProfileId profileId) { public void evict(TenantId tenantId, DeviceProfileId profileId) {
DeviceProfile oldProfile = deviceProfilesMap.remove(profileId); DeviceProfile oldProfile = deviceProfilesMap.remove(profileId);
log.debug("[{}] evict device profile from cache: {}", profileId, oldProfile); log.debug("[{}] evict device profile from cache: {}", profileId, oldProfile);
DeviceProfile newProfile = get(tenantId, profileId); DeviceProfile newProfile = get(tenantId, profileId);
if (newProfile != null) { if (newProfile != null) {
notifyListeners(newProfile); notifyProfileListeners(newProfile);
} }
} }
@Override @Override
public void evict(DeviceId deviceId) { public void evict(TenantId tenantId, DeviceId deviceId) {
devicesMap.remove(deviceId); DeviceProfileId old = devicesMap.remove(deviceId);
if (old != null) {
DeviceProfile newProfile = get(tenantId, deviceId);
if (newProfile == null || !old.equals(newProfile.getId())) {
notifyDeviceListeners(tenantId, deviceId, newProfile);
}
}
} }
@Override @Override
public void addListener(TenantId tenantId, EntityId listenerId, Consumer<DeviceProfile> listener) { public void addListener(TenantId tenantId, EntityId listenerId,
listeners.computeIfAbsent(tenantId, id -> new ConcurrentHashMap<>()).put(listenerId, listener); Consumer<DeviceProfile> profileListener,
BiConsumer<DeviceId, DeviceProfile> deviceListener) {
if (profileListener != null) {
profileListeners.computeIfAbsent(tenantId, id -> new ConcurrentHashMap<>()).put(listenerId, profileListener);
}
if (deviceListener != null) {
deviceProfileListeners.computeIfAbsent(tenantId, id -> new ConcurrentHashMap<>()).put(listenerId, deviceListener);
}
} }
@Override @Override
@ -128,17 +133,30 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache {
@Override @Override
public void removeListener(TenantId tenantId, EntityId listenerId) { public void removeListener(TenantId tenantId, EntityId listenerId) {
ConcurrentMap<EntityId, Consumer<DeviceProfile>> tenantListeners = listeners.get(tenantId); ConcurrentMap<EntityId, Consumer<DeviceProfile>> tenantListeners = profileListeners.get(tenantId);
if (tenantListeners != null) { if (tenantListeners != null) {
tenantListeners.remove(listenerId); tenantListeners.remove(listenerId);
} }
ConcurrentMap<EntityId, BiConsumer<DeviceId, DeviceProfile>> deviceListeners = deviceProfileListeners.get(tenantId);
if (deviceListeners != null) {
deviceListeners.remove(listenerId);
}
} }
private void notifyListeners(DeviceProfile profile) { private void notifyProfileListeners(DeviceProfile profile) {
ConcurrentMap<EntityId, Consumer<DeviceProfile>> tenantListeners = listeners.get(profile.getTenantId()); ConcurrentMap<EntityId, Consumer<DeviceProfile>> tenantListeners = profileListeners.get(profile.getTenantId());
if (tenantListeners != null) { if (tenantListeners != null) {
tenantListeners.forEach((id, listener) -> listener.accept(profile)); tenantListeners.forEach((id, listener) -> listener.accept(profile));
} }
} }
private void notifyDeviceListeners(TenantId tenantId, DeviceId deviceId, DeviceProfile profile) {
if (profile != null) {
ConcurrentMap<EntityId, BiConsumer<DeviceId, DeviceProfile>> tenantListeners = deviceProfileListeners.get(tenantId);
if (tenantListeners != null) {
tenantListeners.forEach((id, listener) -> listener.accept(deviceId, profile));
}
}
}
} }

4
application/src/main/java/org/thingsboard/server/service/profile/TbDeviceProfileCache.java

@ -23,11 +23,9 @@ import org.thingsboard.server.common.data.id.TenantId;
public interface TbDeviceProfileCache extends RuleEngineDeviceProfileCache { public interface TbDeviceProfileCache extends RuleEngineDeviceProfileCache {
void put(DeviceProfile profile);
void evict(TenantId tenantId, DeviceProfileId id); void evict(TenantId tenantId, DeviceProfileId id);
void evict(DeviceId id); void evict(TenantId tenantId, DeviceId id);
DeviceProfile find(DeviceProfileId deviceProfileId); DeviceProfile find(DeviceProfileId deviceProfileId);

25
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java

@ -21,6 +21,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasName;
@ -47,6 +48,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotifica
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueCallback;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.MultipleTbQueueCallbackWrapper;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
@ -214,6 +216,12 @@ public class DefaultTbClusterService implements TbClusterService {
onEntityChange(TenantId.SYS_TENANT_ID, tenant.getId(), tenant, callback); onEntityChange(TenantId.SYS_TENANT_ID, tenant.getId(), tenant, callback);
} }
@Override
public void onApiStateChange(ApiUsageState apiUsageState, TbQueueCallback callback) {
onEntityChange(apiUsageState.getTenantId(), apiUsageState.getId(), apiUsageState, callback);
broadcast(new ComponentLifecycleMsg(apiUsageState.getTenantId(), apiUsageState.getId(), ComponentLifecycleEvent.UPDATED));
}
@Override @Override
public void onDeviceProfileDelete(DeviceProfile entity, TbQueueCallback callback) { public void onDeviceProfileDelete(DeviceProfile entity, TbQueueCallback callback) {
onEntityDelete(entity.getTenantId(), entity.getId(), entity.getName(), callback); onEntityDelete(entity.getTenantId(), entity.getId(), entity.getName(), callback);
@ -229,13 +237,14 @@ public class DefaultTbClusterService implements TbClusterService {
onEntityDelete(TenantId.SYS_TENANT_ID, entity.getId(), entity.getName(), callback); onEntityDelete(TenantId.SYS_TENANT_ID, entity.getId(), entity.getName(), callback);
} }
public <T extends HasName> void onEntityChange(TenantId tenantId, EntityId entityid, T entity, TbQueueCallback callback) { public <T> void onEntityChange(TenantId tenantId, EntityId entityid, T entity, TbQueueCallback callback) {
log.trace("[{}][{}][{}] Processing [{}] change event", tenantId, entityid.getEntityType(), entityid.getId(), entity.getName()); String entityName = (entity instanceof HasName) ? ((HasName) entity).getName() : entity.getClass().getName();
log.trace("[{}][{}][{}] Processing [{}] change event", tenantId, entityid.getEntityType(), entityid.getId(), entityName);
TransportProtos.EntityUpdateMsg entityUpdateMsg = TransportProtos.EntityUpdateMsg.newBuilder() TransportProtos.EntityUpdateMsg entityUpdateMsg = TransportProtos.EntityUpdateMsg.newBuilder()
.setEntityType(entityid.getEntityType().name()) .setEntityType(entityid.getEntityType().name())
.setData(ByteString.copyFrom(encodingService.encode(entity))).build(); .setData(ByteString.copyFrom(encodingService.encode(entity))).build();
ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setEntityUpdateMsg(entityUpdateMsg).build(); ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setEntityUpdateMsg(entityUpdateMsg).build();
broadcast(transportMsg); broadcast(transportMsg, callback);
} }
private void onEntityDelete(TenantId tenantId, EntityId entityId, String name, TbQueueCallback callback) { private void onEntityDelete(TenantId tenantId, EntityId entityId, String name, TbQueueCallback callback) {
@ -246,15 +255,16 @@ public class DefaultTbClusterService implements TbClusterService {
.setEntityIdLSB(entityId.getId().getLeastSignificantBits()) .setEntityIdLSB(entityId.getId().getLeastSignificantBits())
.build(); .build();
ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setEntityDeleteMsg(entityDeleteMsg).build(); ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setEntityDeleteMsg(entityDeleteMsg).build();
broadcast(transportMsg); broadcast(transportMsg, callback);
} }
private void broadcast(ToTransportMsg transportMsg) { private void broadcast(ToTransportMsg transportMsg, TbQueueCallback callback) {
TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> toTransportNfProducer = producerProvider.getTransportNotificationsMsgProducer(); TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> toTransportNfProducer = producerProvider.getTransportNotificationsMsgProducer();
Set<String> tbTransportServices = partitionService.getAllServiceIds(ServiceType.TB_TRANSPORT); Set<String> tbTransportServices = partitionService.getAllServiceIds(ServiceType.TB_TRANSPORT);
TbQueueCallback proxyCallback = callback != null ? new MultipleTbQueueCallbackWrapper(tbTransportServices.size(), callback) : null;
for (String transportServiceId : tbTransportServices) { for (String transportServiceId : tbTransportServices) {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, transportServiceId); TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, transportServiceId);
toTransportNfProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), null); toTransportNfProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), proxyCallback);
toTransportNfs.incrementAndGet(); toTransportNfs.incrementAndGet();
} }
} }
@ -264,7 +274,8 @@ public class DefaultTbClusterService implements TbClusterService {
TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer();
Set<String> tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE)); Set<String> tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE));
if (msg.getEntityId().getEntityType().equals(EntityType.TENANT) if (msg.getEntityId().getEntityType().equals(EntityType.TENANT)
|| msg.getEntityId().getEntityType().equals(EntityType.DEVICE_PROFILE)) { || msg.getEntityId().getEntityType().equals(EntityType.DEVICE_PROFILE)
|| msg.getEntityId().getEntityType().equals(EntityType.API_USAGE_STATE)) {
TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer();
Set<String> tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); Set<String> tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE);
for (String serviceId : tbCoreServices) { for (String serviceId : tbCoreServices) {

111
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -17,41 +17,44 @@ package org.thingsboard.server.service.queue;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.rule.engine.api.RpcError; import org.thingsboard.rule.engine.api.RpcError;
import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.Alarm;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.MsgType;
import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.dao.util.mapping.JacksonUtil; import org.thingsboard.server.dao.util.mapping.JacksonUtil;
import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto;
import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto;
import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto;
import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeDeleteProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmUpdateProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmDeleteProto; import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmDeleteProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmUpdateProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeDeleteProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto; import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto; import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.service.profile.TbTenantProfileCache;
import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService;
@ -70,6 +73,8 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch; import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.function.Function; import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -88,32 +93,56 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
private final TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> mainConsumer; private final TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> mainConsumer;
private final DeviceStateService stateService; private final DeviceStateService stateService;
private final TbApiUsageStateService statsService;
private final TbLocalSubscriptionService localSubscriptionService; private final TbLocalSubscriptionService localSubscriptionService;
private final SubscriptionManagerService subscriptionManagerService; private final SubscriptionManagerService subscriptionManagerService;
private final TbCoreDeviceRpcService tbCoreDeviceRpcService; private final TbCoreDeviceRpcService tbCoreDeviceRpcService;
private final TbCoreConsumerStats stats; private final TbCoreConsumerStats stats;
protected final TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> usageStatsConsumer;
protected volatile ExecutorService usageStatsExecutor;
public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory,
DeviceStateService stateService, TbLocalSubscriptionService localSubscriptionService, ActorSystemContext actorContext,
SubscriptionManagerService subscriptionManagerService, DataDecodingEncodingService encodingService, DeviceStateService stateService,
TbCoreDeviceRpcService tbCoreDeviceRpcService, StatsFactory statsFactory, TbDeviceProfileCache deviceProfileCache) { TbLocalSubscriptionService localSubscriptionService,
super(actorContext, encodingService, deviceProfileCache, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer()); SubscriptionManagerService subscriptionManagerService,
DataDecodingEncodingService encodingService,
TbCoreDeviceRpcService tbCoreDeviceRpcService,
StatsFactory statsFactory,
TbDeviceProfileCache deviceProfileCache,
TbApiUsageStateService statsService,
TbTenantProfileCache tenantProfileCache,
TbApiUsageStateService apiUsageStateService) {
super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, apiUsageStateService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer());
this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer(); this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer();
this.usageStatsConsumer = tbCoreQueueFactory.createToUsageStatsServiceMsgConsumer();
this.stateService = stateService; this.stateService = stateService;
this.localSubscriptionService = localSubscriptionService; this.localSubscriptionService = localSubscriptionService;
this.subscriptionManagerService = subscriptionManagerService; this.subscriptionManagerService = subscriptionManagerService;
this.tbCoreDeviceRpcService = tbCoreDeviceRpcService; this.tbCoreDeviceRpcService = tbCoreDeviceRpcService;
this.stats = new TbCoreConsumerStats(statsFactory); this.stats = new TbCoreConsumerStats(statsFactory);
this.statsService = statsService;
} }
@PostConstruct @PostConstruct
public void init() { public void init() {
super.init("tb-core-consumer", "tb-core-notifications-consumer"); super.init("tb-core-consumer", "tb-core-notifications-consumer");
this.usageStatsExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-core-usage-stats-consumer"));
} }
@PreDestroy @PreDestroy
public void destroy() { public void destroy() {
super.destroy(); super.destroy();
if (usageStatsExecutor != null) {
usageStatsExecutor.shutdownNow();
}
}
@EventListener(ApplicationReadyEvent.class)
public void onApplicationEvent(ApplicationReadyEvent event) {
super.onApplicationEvent(event);
launchUsageStatsConsumer();
} }
@Override @Override
@ -121,6 +150,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
if (partitionChangeEvent.getServiceType().equals(getServiceType())) { if (partitionChangeEvent.getServiceType().equals(getServiceType())) {
log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions());
this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); this.mainConsumer.subscribe(partitionChangeEvent.getPartitions());
this.usageStatsConsumer.subscribe(
partitionChangeEvent
.getPartitions()
.stream()
.map(tpi -> tpi.newByTopic(usageStatsConsumer.getTopic()))
.collect(Collectors.toSet()));
} }
} }
@ -223,6 +258,53 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
} }
} }
private void launchUsageStatsConsumer() {
usageStatsExecutor.submit(() -> {
while (!stopped) {
try {
List<TbProtoQueueMsg<ToUsageStatsServiceMsg>> msgs = usageStatsConsumer.poll(getNotificationPollDuration());
if (msgs.isEmpty()) {
continue;
}
ConcurrentMap<UUID, TbProtoQueueMsg<ToUsageStatsServiceMsg>> pendingMap = msgs.stream().collect(
Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity()));
CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
TbPackProcessingContext<TbProtoQueueMsg<ToUsageStatsServiceMsg>> ctx = new TbPackProcessingContext<>(
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
pendingMap.forEach((id, msg) -> {
log.trace("[{}] Creating usage stats callback for message: {}", id, msg.getValue());
TbCallback callback = new TbPackCallback<>(id, ctx);
try {
handleUsageStats(msg, callback);
} catch (Throwable e) {
log.warn("[{}] Failed to process usge stats: {}", id, msg, e);
callback.onFailure(e);
}
});
if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) {
ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process usage stats: {}", id, msg.getValue()));
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process usage stats: {}", id, msg.getValue()));
}
usageStatsConsumer.commit();
} catch (Exception e) {
if (!stopped) {
log.warn("Failed to obtain usage stats from queue.", e);
try {
Thread.sleep(getNotificationPollDuration());
} catch (InterruptedException e2) {
log.trace("Failed to wait until the server has capacity to handle new usage stats", e2);
}
}
}
}
log.info("TB Usage Stats Consumer stopped.");
});
}
private void handleUsageStats(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback) {
statsService.process(msg, callback);
}
private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbCallback callback) { private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbCallback callback) {
RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null;
FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()) FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB())
@ -321,6 +403,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
if (mainConsumer != null) { if (mainConsumer != null) {
mainConsumer.unsubscribe(); mainConsumer.unsubscribe();
} }
if (usageStatsConsumer != null) {
usageStatsConsumer.unsubscribe();
}
} }
} }

48
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@ -15,7 +15,6 @@
*/ */
package org.thingsboard.server.service.queue; package org.thingsboard.server.service.queue;
import com.google.protobuf.ByteString;
import com.google.protobuf.ProtocolStringList; import com.google.protobuf.ProtocolStringList;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
@ -24,10 +23,16 @@ import org.springframework.stereotype.Service;
import org.thingsboard.rule.engine.api.RpcError; import org.thingsboard.rule.engine.api.RpcError;
import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.queue.*; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg;
import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
import org.thingsboard.server.common.msg.queue.ServiceQueue;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
@ -38,17 +43,33 @@ import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.queue.util.TbRuleEngineComponent;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.service.queue.processing.*; import org.thingsboard.server.service.profile.TbTenantProfileCache;
import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory;
import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService;
import org.thingsboard.server.service.stats.RuleEngineStatisticsService; import org.thingsboard.server.service.stats.RuleEngineStatisticsService;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy; import javax.annotation.PreDestroy;
import java.util.*; import java.util.Collections;
import java.util.concurrent.*; import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@Service @Service
@TbRuleEngineComponent @TbRuleEngineComponent
@ -79,11 +100,16 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory,
TbRuleEngineSubmitStrategyFactory submitStrategyFactory, TbRuleEngineSubmitStrategyFactory submitStrategyFactory,
TbQueueRuleEngineSettings ruleEngineSettings, TbQueueRuleEngineSettings ruleEngineSettings,
TbRuleEngineQueueFactory tbRuleEngineQueueFactory, RuleEngineStatisticsService statisticsService, TbRuleEngineQueueFactory tbRuleEngineQueueFactory,
ActorSystemContext actorContext, DataDecodingEncodingService encodingService, RuleEngineStatisticsService statisticsService,
ActorSystemContext actorContext,
DataDecodingEncodingService encodingService,
TbRuleEngineDeviceRpcService tbDeviceRpcService, TbRuleEngineDeviceRpcService tbDeviceRpcService,
StatsFactory statsFactory, TbDeviceProfileCache deviceProfileCache) { StatsFactory statsFactory,
super(actorContext, encodingService, deviceProfileCache, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer()); TbDeviceProfileCache deviceProfileCache,
TbTenantProfileCache tenantProfileCache,
TbApiUsageStateService apiUsageStateService) {
super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, apiUsageStateService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer());
this.statisticsService = statisticsService; this.statisticsService = statisticsService;
this.ruleEngineSettings = ruleEngineSettings; this.ruleEngineSettings = ruleEngineSettings;
this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory;

3
application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java

@ -16,6 +16,7 @@
package org.thingsboard.server.service.queue; package org.thingsboard.server.service.queue;
import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.TenantProfile;
@ -63,4 +64,6 @@ public interface TbClusterService {
void onTenantChange(Tenant tenant, TbQueueCallback callback); void onTenantChange(Tenant tenant, TbQueueCallback callback);
void onTenantDelete(Tenant tenant, TbQueueCallback callback); void onTenantDelete(Tenant tenant, TbQueueCallback callback);
void onApiStateChange(ApiUsageState apiUsageState, TbQueueCallback callback);
} }

27
application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java

@ -25,6 +25,8 @@ import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.TenantProfileId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
@ -33,7 +35,9 @@ import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.service.profile.TbTenantProfileCache;
import org.thingsboard.server.service.queue.TbPackCallback; import org.thingsboard.server.service.queue.TbPackCallback;
import org.thingsboard.server.service.queue.TbPackProcessingContext; import org.thingsboard.server.service.queue.TbPackProcessingContext;
@ -59,15 +63,19 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
protected final ActorSystemContext actorContext; protected final ActorSystemContext actorContext;
protected final DataDecodingEncodingService encodingService; protected final DataDecodingEncodingService encodingService;
protected final TbTenantProfileCache tenantProfileCache;
protected final TbDeviceProfileCache deviceProfileCache; protected final TbDeviceProfileCache deviceProfileCache;
protected final TbApiUsageStateService apiUsageStateService;
protected final TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer; protected final TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer;
public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService,
TbDeviceProfileCache deviceProfileCache, TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer) { TbTenantProfileCache tenantProfileCache, TbDeviceProfileCache deviceProfileCache, TbApiUsageStateService apiUsageStateService, TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer) {
this.actorContext = actorContext; this.actorContext = actorContext;
this.encodingService = encodingService; this.encodingService = encodingService;
this.tenantProfileCache = tenantProfileCache;
this.deviceProfileCache = deviceProfileCache; this.deviceProfileCache = deviceProfileCache;
this.apiUsageStateService = apiUsageStateService;
this.nfConsumer = nfConsumer; this.nfConsumer = nfConsumer;
} }
@ -143,10 +151,23 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
TbActorMsg actorMsg = actorMsgOpt.get(); TbActorMsg actorMsg = actorMsgOpt.get();
if (actorMsg instanceof ComponentLifecycleMsg) { if (actorMsg instanceof ComponentLifecycleMsg) {
ComponentLifecycleMsg componentLifecycleMsg = (ComponentLifecycleMsg) actorMsg; ComponentLifecycleMsg componentLifecycleMsg = (ComponentLifecycleMsg) actorMsg;
if (EntityType.DEVICE_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { if (EntityType.TENANT_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
TenantProfileId tenantProfileId = new TenantProfileId(componentLifecycleMsg.getEntityId().getId());
tenantProfileCache.evict(tenantProfileId);
if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.UPDATED)) {
apiUsageStateService.onTenantProfileUpdate(tenantProfileId);
}
} else if (EntityType.TENANT.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
tenantProfileCache.evict(componentLifecycleMsg.getTenantId());
if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.UPDATED)) {
apiUsageStateService.onTenantUpdate(componentLifecycleMsg.getTenantId());
}
} else if (EntityType.DEVICE_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceProfileId(componentLifecycleMsg.getEntityId().getId())); deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceProfileId(componentLifecycleMsg.getEntityId().getId()));
} else if (EntityType.DEVICE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { } else if (EntityType.DEVICE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
deviceProfileCache.evict(new DeviceId(componentLifecycleMsg.getEntityId().getId())); deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceId(componentLifecycleMsg.getEntityId().getId()));
} else if (EntityType.API_USAGE_STATE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
apiUsageStateService.onApiUsageStateUpdate(componentLifecycleMsg.getTenantId());
} }
} }
log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg); log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg);

64
application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java

@ -19,6 +19,10 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import java.util.Map; import java.util.Map;
import java.util.UUID; import java.util.UUID;
@ -33,9 +37,16 @@ import java.util.concurrent.atomic.AtomicInteger;
@Slf4j @Slf4j
public abstract class AbstractJsInvokeService implements JsInvokeService { public abstract class AbstractJsInvokeService implements JsInvokeService {
private final TbApiUsageStateService apiUsageStateService;
private final TbApiUsageClient apiUsageClient;
protected ScheduledExecutorService timeoutExecutorService; protected ScheduledExecutorService timeoutExecutorService;
protected Map<UUID, String> scriptIdToNameMap = new ConcurrentHashMap<>(); protected Map<UUID, String> scriptIdToNameMap = new ConcurrentHashMap<>();
protected Map<UUID, BlackListInfo> blackListedFunctions = new ConcurrentHashMap<>(); protected Map<UUID, DisableListInfo> disabledFunctions = new ConcurrentHashMap<>();
protected AbstractJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient) {
this.apiUsageStateService = apiUsageStateService;
this.apiUsageClient = apiUsageClient;
}
public void init(long maxRequestsTimeout) { public void init(long maxRequestsTimeout) {
if (maxRequestsTimeout > 0) { if (maxRequestsTimeout > 0) {
@ -50,24 +61,33 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
} }
@Override @Override
public ListenableFuture<UUID> eval(JsScriptType scriptType, String scriptBody, String... argNames) { public ListenableFuture<UUID> eval(TenantId tenantId, JsScriptType scriptType, String scriptBody, String... argNames) {
UUID scriptId = UUID.randomUUID(); if (apiUsageStateService.getApiUsageState(tenantId).isJsExecEnabled()) {
String functionName = "invokeInternal_" + scriptId.toString().replace('-', '_'); UUID scriptId = UUID.randomUUID();
String jsScript = generateJsScript(scriptType, functionName, scriptBody, argNames); String functionName = "invokeInternal_" + scriptId.toString().replace('-', '_');
return doEval(scriptId, functionName, jsScript); String jsScript = generateJsScript(scriptType, functionName, scriptBody, argNames);
return doEval(scriptId, functionName, jsScript);
} else {
return Futures.immediateFailedFuture(new RuntimeException("JS Execution is disabled due to API limits!"));
}
} }
@Override @Override
public ListenableFuture<Object> invokeFunction(UUID scriptId, Object... args) { public ListenableFuture<Object> invokeFunction(TenantId tenantId, UUID scriptId, Object... args) {
String functionName = scriptIdToNameMap.get(scriptId); if (apiUsageStateService.getApiUsageState(tenantId).isJsExecEnabled()) {
if (functionName == null) { String functionName = scriptIdToNameMap.get(scriptId);
return Futures.immediateFailedFuture(new RuntimeException("No compiled script found for scriptId: [" + scriptId + "]!")); if (functionName == null) {
} return Futures.immediateFailedFuture(new RuntimeException("No compiled script found for scriptId: [" + scriptId + "]!"));
if (!isBlackListed(scriptId)) { }
return doInvokeFunction(scriptId, functionName, args); if (!isDisabled(scriptId)) {
apiUsageClient.report(tenantId, ApiUsageRecordKey.JS_EXEC_COUNT, 1);
return doInvokeFunction(scriptId, functionName, args);
} else {
return Futures.immediateFailedFuture(
new RuntimeException("Script invocation is blocked due to maximum error count " + getMaxErrors() + "!"));
}
} else { } else {
return Futures.immediateFailedFuture( return Futures.immediateFailedFuture(new RuntimeException("JS Execution is disabled due to API limits!"));
new RuntimeException("Script is blacklisted due to maximum error count " + getMaxErrors() + "!"));
} }
} }
@ -77,7 +97,7 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
if (functionName != null) { if (functionName != null) {
try { try {
scriptIdToNameMap.remove(scriptId); scriptIdToNameMap.remove(scriptId);
blackListedFunctions.remove(scriptId); disabledFunctions.remove(scriptId);
doRelease(scriptId, functionName); doRelease(scriptId, functionName);
} catch (Exception e) { } catch (Exception e) {
return Futures.immediateFailedFuture(e); return Futures.immediateFailedFuture(e);
@ -97,7 +117,7 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
protected abstract long getMaxBlacklistDuration(); protected abstract long getMaxBlacklistDuration();
protected void onScriptExecutionError(UUID scriptId) { protected void onScriptExecutionError(UUID scriptId) {
blackListedFunctions.computeIfAbsent(scriptId, key -> new BlackListInfo()).incrementAndGet(); disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()).incrementAndGet();
} }
private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) { private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) {
@ -107,11 +127,11 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); throw new RuntimeException("No script factory implemented for scriptType: " + scriptType);
} }
private boolean isBlackListed(UUID scriptId) { private boolean isDisabled(UUID scriptId) {
BlackListInfo errorCount = blackListedFunctions.get(scriptId); DisableListInfo errorCount = disabledFunctions.get(scriptId);
if (errorCount != null) { if (errorCount != null) {
if (errorCount.getExpirationTime() <= System.currentTimeMillis()) { if (errorCount.getExpirationTime() <= System.currentTimeMillis()) {
blackListedFunctions.remove(scriptId); disabledFunctions.remove(scriptId);
return false; return false;
} else { } else {
return errorCount.get() >= getMaxErrors(); return errorCount.get() >= getMaxErrors();
@ -121,11 +141,11 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
} }
} }
private class BlackListInfo { private class DisableListInfo {
private final AtomicInteger counter; private final AtomicInteger counter;
private long expirationTime; private long expirationTime;
private BlackListInfo() { private DisableListInfo() {
this.counter = new AtomicInteger(0); this.counter = new AtomicInteger(0);
} }

13
application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java

@ -24,10 +24,10 @@ import delight.nashornsandbox.NashornSandboxes;
import jdk.nashorn.api.scripting.NashornScriptEngineFactory; import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
import lombok.Getter; import lombok.Getter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy; import javax.annotation.PreDestroy;
@ -38,7 +38,6 @@ import java.util.UUID;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
@ -57,9 +56,8 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
private final FutureCallback<UUID> evalCallback = new JsStatCallback<>(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs); private final FutureCallback<UUID> evalCallback = new JsStatCallback<>(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs);
private final FutureCallback<Object> invokeCallback = new JsStatCallback<>(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs); private final FutureCallback<Object> invokeCallback = new JsStatCallback<>(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs);
@Autowired
@Getter @Getter
private JsExecutorService jsExecutor; private final JsExecutorService jsExecutor;
@Value("${js.local.max_requests_timeout:0}") @Value("${js.local.max_requests_timeout:0}")
private long maxRequestsTimeout; private long maxRequestsTimeout;
@ -67,6 +65,11 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
@Value("${js.local.stats.enabled:false}") @Value("${js.local.stats.enabled:false}")
private boolean statsEnabled; private boolean statsEnabled;
public AbstractNashornJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient, JsExecutorService jsExecutor) {
super(apiUsageStateService, apiUsageClient);
this.jsExecutor = jsExecutor;
}
@Scheduled(fixedDelayString = "${js.local.stats.print_interval_ms:10000}") @Scheduled(fixedDelayString = "${js.local.stats.print_interval_ms:10000}")
public void printStats() { public void printStats() {
if (statsEnabled) { if (statsEnabled) {

2
application/src/main/java/org/thingsboard/server/service/script/JsExecutorService.java

@ -27,7 +27,7 @@ public class JsExecutorService extends AbstractListeningExecutor {
@Override @Override
protected int getThreadPollSize() { protected int getThreadPollSize() {
return jsExecutorThreadPoolSize; return Math.max(jsExecutorThreadPoolSize, 1);
} }
} }

5
application/src/main/java/org/thingsboard/server/service/script/JsInvokeService.java

@ -17,14 +17,15 @@ package org.thingsboard.server.service.script;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import java.util.UUID; import java.util.UUID;
public interface JsInvokeService { public interface JsInvokeService {
ListenableFuture<UUID> eval(JsScriptType scriptType, String scriptBody, String... argNames); ListenableFuture<UUID> eval(TenantId tenantId, JsScriptType scriptType, String scriptBody, String... argNames);
ListenableFuture<Object> invokeFunction(UUID scriptId, Object... args); ListenableFuture<Object> invokeFunction(TenantId tenantId, UUID scriptId, Object... args);
ListenableFuture<Void> release(UUID scriptId); ListenableFuture<Void> release(UUID scriptId);

6
application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java

@ -19,6 +19,8 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -42,6 +44,10 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService {
@Value("${js.local.max_black_list_duration_sec:60}") @Value("${js.local.max_black_list_duration_sec:60}")
private int maxBlackListDurationSec; private int maxBlackListDurationSec;
public NashornJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient, JsExecutorService jsExecutor) {
super(apiUsageStateService, apiUsageClient, jsExecutor);
}
@Override @Override
protected boolean useJsSandbox() { protected boolean useJsSandbox() {
return useJsSandbox; return useJsSandbox;

6
application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java

@ -30,6 +30,8 @@ import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
@ -68,6 +70,10 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
private final AtomicInteger queueFailedMsgs = new AtomicInteger(0); private final AtomicInteger queueFailedMsgs = new AtomicInteger(0);
private final AtomicInteger queueTimeoutMsgs = new AtomicInteger(0); private final AtomicInteger queueTimeoutMsgs = new AtomicInteger(0);
public RemoteJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient) {
super(apiUsageStateService, apiUsageClient);
}
@Scheduled(fixedDelayString = "${js.remote.stats.print_interval_ms}") @Scheduled(fixedDelayString = "${js.remote.stats.print_interval_ms}")
public void printStats() { public void printStats() {
if (statsEnabled) { if (statsEnabled) {

11
application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java

@ -25,6 +25,7 @@ import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.TbMsgMetaData;
@ -43,13 +44,15 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
private final JsInvokeService sandboxService; private final JsInvokeService sandboxService;
private final UUID scriptId; private final UUID scriptId;
private final TenantId tenantId;
private final EntityId entityId; private final EntityId entityId;
public RuleNodeJsScriptEngine(JsInvokeService sandboxService, EntityId entityId, String script, String... argNames) { public RuleNodeJsScriptEngine(TenantId tenantId, JsInvokeService sandboxService, EntityId entityId, String script, String... argNames) {
this.tenantId = tenantId;
this.sandboxService = sandboxService; this.sandboxService = sandboxService;
this.entityId = entityId; this.entityId = entityId;
try { try {
this.scriptId = this.sandboxService.eval(JsScriptType.RULE_NODE_SCRIPT, script, argNames).get(); this.scriptId = this.sandboxService.eval(tenantId, JsScriptType.RULE_NODE_SCRIPT, script, argNames).get();
} catch (Exception e) { } catch (Exception e) {
Throwable t = e; Throwable t = e;
if (e instanceof ExecutionException) { if (e instanceof ExecutionException) {
@ -203,7 +206,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
private JsonNode executeScript(TbMsg msg) throws ScriptException { private JsonNode executeScript(TbMsg msg) throws ScriptException {
try { try {
String[] inArgs = prepareArgs(msg); String[] inArgs = prepareArgs(msg);
String eval = sandboxService.invokeFunction(this.scriptId, inArgs[0], inArgs[1], inArgs[2]).get().toString(); String eval = sandboxService.invokeFunction(tenantId, this.scriptId, inArgs[0], inArgs[1], inArgs[2]).get().toString();
return mapper.readTree(eval); return mapper.readTree(eval);
} catch (ExecutionException e) { } catch (ExecutionException e) {
if (e.getCause() instanceof ScriptException) { if (e.getCause() instanceof ScriptException) {
@ -220,7 +223,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
private ListenableFuture<JsonNode> executeScriptAsync(TbMsg msg) { private ListenableFuture<JsonNode> executeScriptAsync(TbMsg msg) {
String[] inArgs = prepareArgs(msg); String[] inArgs = prepareArgs(msg);
return Futures.transformAsync(sandboxService.invokeFunction(this.scriptId, inArgs[0], inArgs[1], inArgs[2]), return Futures.transformAsync(sandboxService.invokeFunction(tenantId, this.scriptId, inArgs[0], inArgs[1], inArgs[2]),
o -> { o -> {
try { try {
return Futures.immediateFuture(mapper.readTree(o.toString())); return Futures.immediateFuture(mapper.readTree(o.toString()));

30
application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java

@ -25,6 +25,7 @@ import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.web.context.request.async.DeferredResult; import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfile;
@ -33,6 +34,7 @@ import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.ApiUsageStateId;
import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceId;
@ -55,6 +57,7 @@ import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.AccessControlService;
@ -111,6 +114,9 @@ public class AccessValidator {
@Autowired @Autowired
protected AccessControlService accessControlService; protected AccessControlService accessControlService;
@Autowired
protected ApiUsageStateService apiUsageStateService;
private ExecutorService executor; private ExecutorService executor;
@PostConstruct @PostConstruct
@ -193,6 +199,9 @@ public class AccessValidator {
case ENTITY_VIEW: case ENTITY_VIEW:
validateEntityView(currentUser, operation, entityId, callback); validateEntityView(currentUser, operation, entityId, callback);
return; return;
case API_USAGE_STATE:
validateApiUsageState(currentUser, operation, entityId, callback);
return;
default: default:
//TODO: add support of other entities //TODO: add support of other entities
throw new IllegalStateException("Not Implemented!"); throw new IllegalStateException("Not Implemented!");
@ -237,6 +246,27 @@ public class AccessValidator {
} }
} }
private void validateApiUsageState(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback<ValidationResult> callback) {
if (currentUser.isSystemAdmin()) {
callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION));
} else {
if (!operation.equals(Operation.READ_TELEMETRY)) {
callback.onSuccess(ValidationResult.accessDenied("Allowed only READ_TELEMETRY operation!"));
}
ApiUsageState apiUsageState = apiUsageStateService.findApiUsageStateById(currentUser.getTenantId(), new ApiUsageStateId(entityId.getId()));
if (apiUsageState == null) {
callback.onSuccess(ValidationResult.entityNotFound("Api Usage State with requested id wasn't found!"));
} else {
try {
accessControlService.checkPermission(currentUser, Resource.API_USAGE_STATE, operation, entityId, apiUsageState);
} catch (ThingsboardException e) {
callback.onSuccess(ValidationResult.accessDenied(e.getMessage()));
}
callback.onSuccess(ValidationResult.ok(apiUsageState));
}
}
}
private void validateAsset(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback<ValidationResult> callback) { private void validateAsset(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback<ValidationResult> callback) {
if (currentUser.isSystemAdmin()) { if (currentUser.isSystemAdmin()) {
callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION)); callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION));

3
application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java

@ -35,7 +35,8 @@ public enum Resource {
OAUTH2_CONFIGURATION_INFO(), OAUTH2_CONFIGURATION_INFO(),
OAUTH2_CONFIGURATION_TEMPLATE(), OAUTH2_CONFIGURATION_TEMPLATE(),
TENANT_PROFILE(EntityType.TENANT_PROFILE), TENANT_PROFILE(EntityType.TENANT_PROFILE),
DEVICE_PROFILE(EntityType.DEVICE_PROFILE); DEVICE_PROFILE(EntityType.DEVICE_PROFILE),
API_USAGE_STATE(EntityType.API_USAGE_STATE);
private final EntityType entityType; private final EntityType entityType;

1
application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java

@ -40,6 +40,7 @@ public class TenantAdminPermissions extends AbstractPermissions {
put(Resource.WIDGETS_BUNDLE, widgetsPermissionChecker); put(Resource.WIDGETS_BUNDLE, widgetsPermissionChecker);
put(Resource.WIDGET_TYPE, widgetsPermissionChecker); put(Resource.WIDGET_TYPE, widgetsPermissionChecker);
put(Resource.DEVICE_PROFILE, tenantEntityPermissionChecker); put(Resource.DEVICE_PROFILE, tenantEntityPermissionChecker);
put(Resource.API_USAGE_STATE, tenantEntityPermissionChecker);
} }
public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() {

8
application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java

@ -48,9 +48,9 @@ import java.util.stream.Collectors;
public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService { public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService {
public static final String TB_SERVICE_QUEUE = "TbServiceQueue"; public static final String TB_SERVICE_QUEUE = "TbServiceQueue";
public static final FutureCallback<Void> CALLBACK = new FutureCallback<Void>() { public static final FutureCallback<Integer> CALLBACK = new FutureCallback<Integer>() {
@Override @Override
public void onSuccess(@Nullable Void result) { public void onSuccess(@Nullable Integer result) {
} }
@ -85,7 +85,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS
.map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get())))
.collect(Collectors.toList()); .collect(Collectors.toList());
if (!tsList.isEmpty()) { if (!tsList.isEmpty()) {
tsService.saveAndNotify(tenantId, serviceAssetId, tsList, CALLBACK); tsService.saveAndNotifyInternal(tenantId, serviceAssetId, tsList, CALLBACK);
} }
} }
} catch (DataValidationException e) { } catch (DataValidationException e) {
@ -97,7 +97,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS
ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> {
TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString())); TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString()));
try { try {
tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); tsService.saveAndNotifyInternal(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK);
} catch (DataValidationException e2) { } catch (DataValidationException e2) {
if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) {
throw e2; throw e2;

6
application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java

@ -105,10 +105,10 @@ public abstract class AbstractSubscriptionService implements ApplicationListener
} }
} }
protected void addWsCallback(ListenableFuture<List<Void>> saveFuture, Consumer<Void> callback) { protected <T> void addWsCallback(ListenableFuture<T> saveFuture, Consumer<T> callback) {
Futures.addCallback(saveFuture, new FutureCallback<List<Void>>() { Futures.addCallback(saveFuture, new FutureCallback<T>() {
@Override @Override
public void onSuccess(@Nullable List<Void> result) { public void onSuccess(@Nullable T result) {
callback.accept(null); callback.accept(null);
} }

109
application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java

@ -20,10 +20,9 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityView;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
@ -41,11 +40,12 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.queue.TbClusterService;
import org.thingsboard.server.service.subscription.SubscriptionManagerService;
import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import org.thingsboard.server.service.subscription.TbSubscriptionUtils;
import javax.annotation.Nullable; import javax.annotation.Nullable;
@ -59,12 +59,8 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.function.Consumer;
import java.util.stream.Collectors;
/** /**
* Created by ashvayka on 27.03.18. * Created by ashvayka on 27.03.18.
@ -76,6 +72,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
private final AttributesService attrService; private final AttributesService attrService;
private final TimeseriesService tsService; private final TimeseriesService tsService;
private final EntityViewService entityViewService; private final EntityViewService entityViewService;
private final TbApiUsageClient apiUsageClient;
private final TbApiUsageStateService apiUsageStateService;
private ExecutorService tsCallBackExecutor; private ExecutorService tsCallBackExecutor;
@ -83,11 +81,15 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
TimeseriesService tsService, TimeseriesService tsService,
EntityViewService entityViewService, EntityViewService entityViewService,
TbClusterService clusterService, TbClusterService clusterService,
PartitionService partitionService) { PartitionService partitionService,
TbApiUsageClient apiUsageClient,
TbApiUsageStateService apiUsageStateService) {
super(clusterService, partitionService); super(clusterService, partitionService);
this.attrService = attrService; this.attrService = attrService;
this.tsService = tsService; this.tsService = tsService;
this.entityViewService = entityViewService; this.entityViewService = entityViewService;
this.apiUsageClient = apiUsageClient;
this.apiUsageStateService = apiUsageStateService;
} }
@PostConstruct @PostConstruct
@ -116,7 +118,35 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
@Override @Override
public void saveAndNotify(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, long ttl, FutureCallback<Void> callback) { public void saveAndNotify(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, long ttl, FutureCallback<Void> callback) {
ListenableFuture<List<Void>> saveFuture = tsService.save(tenantId, entityId, ts, ttl); checkInternalEntity(entityId);
if (apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) {
saveAndNotifyInternal(tenantId, entityId, ts, ttl, new FutureCallback<Integer>() {
@Override
public void onSuccess(Integer result) {
if (result != null && result > 0) {
apiUsageClient.report(tenantId, ApiUsageRecordKey.STORAGE_DP_COUNT, result);
}
callback.onSuccess(null);
}
@Override
public void onFailure(Throwable t) {
callback.onFailure(t);
}
});
} else{
callback.onFailure(new RuntimeException("DB storage writes are disabled due to API limits!"));
}
}
@Override
public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Integer> callback) {
saveAndNotifyInternal(tenantId, entityId, ts, 0L, callback);
}
@Override
public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, long ttl, FutureCallback<Integer> callback) {
ListenableFuture<Integer> saveFuture = tsService.save(tenantId, entityId, ts, ttl);
addMainCallback(saveFuture, callback); addMainCallback(saveFuture, callback);
addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts));
if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) {
@ -141,9 +171,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
Optional<TsKvEntry> tsKvEntry = entries.stream() Optional<TsKvEntry> tsKvEntry = entries.stream()
.filter(entry -> entry.getTs() > startTs && entry.getTs() <= endTs) .filter(entry -> entry.getTs() > startTs && entry.getTs() <= endTs)
.max(Comparator.comparingLong(TsKvEntry::getTs)); .max(Comparator.comparingLong(TsKvEntry::getTs));
if (tsKvEntry.isPresent()) { tsKvEntry.ifPresent(entityViewLatest::add);
entityViewLatest.add(tsKvEntry.get());
}
} }
} }
if (!entityViewLatest.isEmpty()) { if (!entityViewLatest.isEmpty()) {
@ -176,29 +204,53 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
@Override @Override
public void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, boolean notifyDevice, FutureCallback<Void> callback) { public void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, boolean notifyDevice, FutureCallback<Void> callback) {
checkInternalEntity(entityId);
saveAndNotifyInternal(tenantId, entityId, scope, attributes, notifyDevice, callback);
}
@Override
public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, boolean notifyDevice, FutureCallback<Void> callback) {
ListenableFuture<List<Void>> saveFuture = attrService.save(tenantId, entityId, scope, attributes); ListenableFuture<List<Void>> saveFuture = attrService.save(tenantId, entityId, scope, attributes);
addMainCallback(saveFuture, callback); addVoidCallback(saveFuture, callback);
addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice));
} }
@Override @Override
public void saveLatestAndNotify(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Void> callback) { public void saveLatestAndNotify(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Void> callback) {
checkInternalEntity(entityId);
saveLatestAndNotifyInternal(tenantId, entityId, ts, callback);
}
@Override
public void saveLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Void> callback) {
ListenableFuture<List<Void>> saveFuture = tsService.saveLatest(tenantId, entityId, ts); ListenableFuture<List<Void>> saveFuture = tsService.saveLatest(tenantId, entityId, ts);
addMainCallback(saveFuture, callback); addVoidCallback(saveFuture, callback);
addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts));
} }
@Override @Override
public void deleteAndNotify(TenantId tenantId, EntityId entityId, String scope, List<String> keys, FutureCallback<Void> callback) { public void deleteAndNotify(TenantId tenantId, EntityId entityId, String scope, List<String> keys, FutureCallback<Void> callback) {
checkInternalEntity(entityId);
deleteAndNotifyInternal(tenantId, entityId, scope, keys, callback);
}
@Override
public void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List<String> keys, FutureCallback<Void> callback) {
ListenableFuture<List<Void>> deleteFuture = attrService.removeAll(tenantId, entityId, scope, keys); ListenableFuture<List<Void>> deleteFuture = attrService.removeAll(tenantId, entityId, scope, keys);
addMainCallback(deleteFuture, callback); addVoidCallback(deleteFuture, callback);
addWsCallback(deleteFuture, success -> onAttributesDelete(tenantId, entityId, scope, keys)); addWsCallback(deleteFuture, success -> onAttributesDelete(tenantId, entityId, scope, keys));
} }
@Override @Override
public void deleteLatest(TenantId tenantId, EntityId entityId, List<String> keys, FutureCallback<Void> callback) { public void deleteLatest(TenantId tenantId, EntityId entityId, List<String> keys, FutureCallback<Void> callback) {
checkInternalEntity(entityId);
deleteLatestInternal(tenantId, entityId, keys, callback);
}
@Override
public void deleteLatestInternal(TenantId tenantId, EntityId entityId, List<String> keys, FutureCallback<Void> callback) {
ListenableFuture<List<Void>> deleteFuture = tsService.removeLatest(tenantId, entityId, keys); ListenableFuture<List<Void>> deleteFuture = tsService.removeLatest(tenantId, entityId, keys);
addMainCallback(deleteFuture, callback); addVoidCallback(deleteFuture, callback);
} }
@Override @Override
@ -283,7 +335,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
} }
} }
private <S, R> void addMainCallback(ListenableFuture<S> saveFuture, final FutureCallback<R> callback) { private <S> void addVoidCallback(ListenableFuture<S> saveFuture, final FutureCallback<Void> callback) {
Futures.addCallback(saveFuture, new FutureCallback<S>() { Futures.addCallback(saveFuture, new FutureCallback<S>() {
@Override @Override
public void onSuccess(@Nullable S result) { public void onSuccess(@Nullable S result) {
@ -296,4 +348,25 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
} }
}, tsCallBackExecutor); }, tsCallBackExecutor);
} }
private <S> void addMainCallback(ListenableFuture<S> saveFuture, final FutureCallback<S> callback) {
Futures.addCallback(saveFuture, new FutureCallback<S>() {
@Override
public void onSuccess(@Nullable S result) {
callback.onSuccess(result);
}
@Override
public void onFailure(Throwable t) {
callback.onFailure(t);
}
}, tsCallBackExecutor);
}
private void checkInternalEntity(EntityId entityId) {
if (EntityType.API_USAGE_STATE.equals(entityId.getEntityType())) {
throw new RuntimeException("Can't update API Usage State!");
}
}
} }

46
application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java

@ -0,0 +1,46 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.telemetry;
import com.google.common.util.concurrent.FutureCallback;
import org.thingsboard.rule.engine.api.RuleEngineTelemetryService;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import java.util.List;
/**
* Created by ashvayka on 27.03.18.
*/
public interface InternalTelemetryService extends RuleEngineTelemetryService {
void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Integer> callback);
void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, long ttl, FutureCallback<Integer> callback);
void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, boolean notifyDevice, FutureCallback<Void> callback);
void saveLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Void> callback);
void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List<String> keys, FutureCallback<Void> callback);
void deleteLatestInternal(TenantId tenantId, EntityId entityId, List<String> keys, FutureCallback<Void> callback);
}

2
application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java

@ -22,6 +22,6 @@ import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
/** /**
* Created by ashvayka on 27.03.18. * Created by ashvayka on 27.03.18.
*/ */
public interface TelemetrySubscriptionService extends RuleEngineTelemetryService, ApplicationListener<PartitionChangeEvent> { public interface TelemetrySubscriptionService extends InternalTelemetryService, ApplicationListener<PartitionChangeEvent> {
} }

20
application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java

@ -25,6 +25,7 @@ import com.google.protobuf.ByteString;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils; import org.springframework.util.StringUtils;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfile;
@ -51,7 +52,6 @@ import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.device.provision.ProvisionRequest; import org.thingsboard.server.dao.device.provision.ProvisionRequest;
import org.thingsboard.server.dao.device.provision.ProvisionResponse; import org.thingsboard.server.dao.device.provision.ProvisionResponse;
import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.util.mapping.JacksonUtil; import org.thingsboard.server.dao.util.mapping.JacksonUtil;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto;
@ -68,6 +68,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509Ce
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.dao.device.provision.ProvisionFailedException; import org.thingsboard.server.dao.device.provision.ProvisionFailedException;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.service.profile.TbTenantProfileCache; import org.thingsboard.server.service.profile.TbTenantProfileCache;
@ -92,6 +93,7 @@ public class DefaultTransportApiService implements TransportApiService {
private final TbDeviceProfileCache deviceProfileCache; private final TbDeviceProfileCache deviceProfileCache;
private final TbTenantProfileCache tenantProfileCache; private final TbTenantProfileCache tenantProfileCache;
private final TbApiUsageStateService apiUsageStateService;
private final DeviceService deviceService; private final DeviceService deviceService;
private final RelationService relationService; private final RelationService relationService;
private final DeviceCredentialsService deviceCredentialsService; private final DeviceCredentialsService deviceCredentialsService;
@ -104,13 +106,14 @@ public class DefaultTransportApiService implements TransportApiService {
private final ConcurrentMap<String, ReentrantLock> deviceCreationLocks = new ConcurrentHashMap<>(); private final ConcurrentMap<String, ReentrantLock> deviceCreationLocks = new ConcurrentHashMap<>();
public DefaultTransportApiService(TbDeviceProfileCache deviceProfileCache, public DefaultTransportApiService(TbDeviceProfileCache deviceProfileCache,
TbTenantProfileCache tenantProfileCache, DeviceService deviceService, TbTenantProfileCache tenantProfileCache, TbApiUsageStateService apiUsageStateService, DeviceService deviceService,
RelationService relationService, DeviceCredentialsService deviceCredentialsService, RelationService relationService, DeviceCredentialsService deviceCredentialsService,
DeviceStateService deviceStateService, DbCallbackExecutorService dbCallbackExecutorService, DeviceStateService deviceStateService, DbCallbackExecutorService dbCallbackExecutorService,
TbClusterService tbClusterService, DataDecodingEncodingService dataDecodingEncodingService, TbClusterService tbClusterService, DataDecodingEncodingService dataDecodingEncodingService,
DeviceProvisionService deviceProvisionService) { DeviceProvisionService deviceProvisionService) {
this.deviceProfileCache = deviceProfileCache; this.deviceProfileCache = deviceProfileCache;
this.tenantProfileCache = tenantProfileCache; this.tenantProfileCache = tenantProfileCache;
this.apiUsageStateService = apiUsageStateService;
this.deviceService = deviceService; this.deviceService = deviceService;
this.relationService = relationService; this.relationService = relationService;
this.deviceCredentialsService = deviceCredentialsService; this.deviceCredentialsService = deviceCredentialsService;
@ -316,18 +319,21 @@ public class DefaultTransportApiService implements TransportApiService {
private ListenableFuture<TransportApiResponseMsg> handle(GetEntityProfileRequestMsg requestMsg) { private ListenableFuture<TransportApiResponseMsg> handle(GetEntityProfileRequestMsg requestMsg) {
EntityType entityType = EntityType.valueOf(requestMsg.getEntityType()); EntityType entityType = EntityType.valueOf(requestMsg.getEntityType());
UUID entityUuid = new UUID(requestMsg.getEntityIdMSB(), requestMsg.getEntityIdLSB()); UUID entityUuid = new UUID(requestMsg.getEntityIdMSB(), requestMsg.getEntityIdLSB());
ByteString data; GetEntityProfileResponseMsg.Builder builder = GetEntityProfileResponseMsg.newBuilder();
if (entityType.equals(EntityType.DEVICE_PROFILE)) { if (entityType.equals(EntityType.DEVICE_PROFILE)) {
DeviceProfileId deviceProfileId = new DeviceProfileId(entityUuid); DeviceProfileId deviceProfileId = new DeviceProfileId(entityUuid);
DeviceProfile deviceProfile = deviceProfileCache.find(deviceProfileId); DeviceProfile deviceProfile = deviceProfileCache.find(deviceProfileId);
data = ByteString.copyFrom(dataDecodingEncodingService.encode(deviceProfile)); builder.setData(ByteString.copyFrom(dataDecodingEncodingService.encode(deviceProfile)));
} else if (entityType.equals(EntityType.TENANT)) { } else if (entityType.equals(EntityType.TENANT)) {
TenantProfile tenantProfile = tenantProfileCache.get(new TenantId(entityUuid)); TenantId tenantId = new TenantId(entityUuid);
data = ByteString.copyFrom(dataDecodingEncodingService.encode(tenantProfile)); TenantProfile tenantProfile = tenantProfileCache.get(tenantId);
ApiUsageState state = apiUsageStateService.getApiUsageState(tenantId);
builder.setData(ByteString.copyFrom(dataDecodingEncodingService.encode(tenantProfile)));
builder.setApiState(ByteString.copyFrom(dataDecodingEncodingService.encode(state)));
} else { } else {
throw new RuntimeException("Invalid entity profile request: " + entityType); throw new RuntimeException("Invalid entity profile request: " + entityType);
} }
return Futures.immediateFuture(TransportApiResponseMsg.newBuilder().setEntityProfileResponseMsg(GetEntityProfileResponseMsg.newBuilder().setData(data).build()).build()); return Futures.immediateFuture(TransportApiResponseMsg.newBuilder().setEntityProfileResponseMsg(builder).build());
} }
private ListenableFuture<TransportApiResponseMsg> getDeviceInfo(DeviceId deviceId, DeviceCredentials credentials) { private ListenableFuture<TransportApiResponseMsg> getDeviceInfo(DeviceId deviceId, DeviceCredentials credentials) {

1
application/src/main/resources/thingsboard.yml

@ -673,6 +673,7 @@ queue:
poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}"
partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}"
pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}"
usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}"
stats: stats:
enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}"
print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}"

7
application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java

@ -23,7 +23,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityInfo;
import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
@ -285,7 +286,9 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController
TenantProfile tenantProfile = new TenantProfile(); TenantProfile tenantProfile = new TenantProfile();
tenantProfile.setName(name); tenantProfile.setName(name);
tenantProfile.setDescription(name + " Test"); tenantProfile.setDescription(name + " Test");
tenantProfile.setProfileData(new TenantProfileData()); TenantProfileData tenantProfileData = new TenantProfileData();
tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration());
tenantProfile.setProfileData(tenantProfileData);
tenantProfile.setDefault(false); tenantProfile.setDefault(false);
tenantProfile.setIsolatedTbCore(false); tenantProfile.setIsolatedTbCore(false);
tenantProfile.setIsolatedTbRuleEngine(false); tenantProfile.setIsolatedTbRuleEngine(false);

2
application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java

@ -27,7 +27,7 @@ import java.util.Arrays;
@RunWith(ClasspathSuite.class) @RunWith(ClasspathSuite.class)
@ClasspathSuite.ClassnameFilters({ @ClasspathSuite.ClassnameFilters({
// "org.thingsboard.server.controller.sql.WebsocketApiSqlTest", // "org.thingsboard.server.controller.sql.WebsocketApiSqlTest",
// "org.thingsboard.server.controller.sql.EntityQueryControllerSqlTest", // "org.thingsboard.server.controller.sql.TenantProfileControllerSqlTest",
"org.thingsboard.server.controller.sql.*Test", "org.thingsboard.server.controller.sql.*Test",
}) })
public class ControllerSqlTestSuite { public class ControllerSqlTestSuite {

250
application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java

@ -1,250 +0,0 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.script;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import com.google.common.collect.Sets;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.rule.engine.api.ScriptEngine;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgDataType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import javax.script.ScriptException;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.Assert.*;
public class RuleNodeJsScriptEngineTest {
private ScriptEngine scriptEngine;
private TestNashornJsInvokeService jsSandboxService;
private EntityId ruleNodeId = new RuleNodeId(Uuids.timeBased());
@Before
public void beforeTest() throws Exception {
jsSandboxService = new TestNashornJsInvokeService(false, 1, 100, 3);
}
@After
public void afterTest() throws Exception {
jsSandboxService.stop();
}
@Test
public void msgCanBeUpdated() throws ScriptException {
String function = "metadata.temp = metadata.temp * 10; return {metadata: metadata};";
scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function);
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("temp", "7");
metaData.putValue("humidity", "99");
String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}";
TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson);
TbMsg actual = scriptEngine.executeUpdate(msg);
assertEquals("70", actual.getMetaData().getValue("temp"));
scriptEngine.destroy();
}
@Test
public void newAttributesCanBeAddedInMsg() throws ScriptException {
String function = "metadata.newAttr = metadata.humidity - msg.passed; return {metadata: metadata};";
scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function);
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("temp", "7");
metaData.putValue("humidity", "99");
String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}";
TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson);
TbMsg actual = scriptEngine.executeUpdate(msg);
assertEquals("94", actual.getMetaData().getValue("newAttr"));
scriptEngine.destroy();
}
@Test
public void payloadCanBeUpdated() throws ScriptException {
String function = "msg.passed = msg.passed * metadata.temp; msg.bigObj.newProp = 'Ukraine'; return {msg: msg};";
scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function);
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("temp", "7");
metaData.putValue("humidity", "99");
String rawJson = "{\"name\":\"Vit\",\"passed\": 5,\"bigObj\":{\"prop\":42}}";
TbMsg msg =TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson);
TbMsg actual = scriptEngine.executeUpdate(msg);
String expectedJson = "{\"name\":\"Vit\",\"passed\":35,\"bigObj\":{\"prop\":42,\"newProp\":\"Ukraine\"}}";
assertEquals(expectedJson, actual.getData());
scriptEngine.destroy();
}
@Test
public void metadataAccessibleForFilter() throws ScriptException {
String function = "return metadata.humidity < 15;";
scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function);
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("temp", "7");
metaData.putValue("humidity", "99");
String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}";
TbMsg msg = TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson);
assertFalse(scriptEngine.executeFilter(msg));
scriptEngine.destroy();
}
@Test
public void dataAccessibleForFilter() throws ScriptException {
String function = "return msg.passed < 15 && msg.name === 'Vit' && metadata.temp == 7 && msg.bigObj.prop == 42;";
scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function);
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("temp", "7");
metaData.putValue("humidity", "99");
String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}";
TbMsg msg = TbMsg.newMsg( "USER", null, metaData,TbMsgDataType.JSON, rawJson);
assertTrue(scriptEngine.executeFilter(msg));
scriptEngine.destroy();
}
@Test
public void dataAccessibleForSwitch() throws ScriptException {
String jsCode = "function nextRelation(metadata, msg) {\n" +
" if(msg.passed == 5 && metadata.temp == 10)\n" +
" return 'one'\n" +
" else\n" +
" return 'two';\n" +
"};\n" +
"\n" +
"return nextRelation(metadata, msg);";
scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, jsCode);
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("temp", "10");
metaData.putValue("humidity", "99");
String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}";
TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson);
Set<String> actual = scriptEngine.executeSwitch(msg);
assertEquals(Sets.newHashSet("one"), actual);
scriptEngine.destroy();
}
@Test
public void multipleRelationsReturnedFromSwitch() throws ScriptException {
String jsCode = "function nextRelation(metadata, msg) {\n" +
" if(msg.passed == 5 && metadata.temp == 10)\n" +
" return ['three', 'one']\n" +
" else\n" +
" return 'two';\n" +
"};\n" +
"\n" +
"return nextRelation(metadata, msg);";
scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, jsCode);
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("temp", "10");
metaData.putValue("humidity", "99");
String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}";
TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson);
Set<String> actual = scriptEngine.executeSwitch(msg);
assertEquals(Sets.newHashSet("one", "three"), actual);
scriptEngine.destroy();
}
@Test
public void concurrentReleasedCorrectly() throws InterruptedException, ExecutionException {
String code = "metadata.temp = metadata.temp * 10; return {metadata: metadata};";
int repeat = 1000;
ExecutorService service = Executors.newFixedThreadPool(repeat);
Map<UUID, Object> scriptIds = new ConcurrentHashMap<>();
CountDownLatch startLatch = new CountDownLatch(repeat);
CountDownLatch finishLatch = new CountDownLatch(repeat);
AtomicInteger failedCount = new AtomicInteger(0);
for (int i = 0; i < repeat; i++) {
service.submit(() -> runScript(startLatch, finishLatch, failedCount, scriptIds, code));
}
finishLatch.await();
assertTrue(scriptIds.size() == 1);
assertTrue(failedCount.get() == 0);
CountDownLatch nextStart = new CountDownLatch(repeat);
CountDownLatch nextFinish = new CountDownLatch(repeat);
for (int i = 0; i < repeat; i++) {
service.submit(() -> runScript(nextStart, nextFinish, failedCount, scriptIds, code));
}
nextFinish.await();
assertTrue(scriptIds.size() == 1);
assertTrue(failedCount.get() == 0);
service.shutdownNow();
}
@Test
public void concurrentFailedEvaluationShouldThrowException() throws InterruptedException {
String code = "metadata.temp = metadata.temp * 10; urn {metadata: metadata};";
int repeat = 10000;
ExecutorService service = Executors.newFixedThreadPool(repeat);
Map<UUID, Object> scriptIds = new ConcurrentHashMap<>();
CountDownLatch startLatch = new CountDownLatch(repeat);
CountDownLatch finishLatch = new CountDownLatch(repeat);
AtomicInteger failedCount = new AtomicInteger(0);
for (int i = 0; i < repeat; i++) {
service.submit(() -> {
service.submit(() -> runScript(startLatch, finishLatch, failedCount, scriptIds, code));
});
}
finishLatch.await();
assertTrue(scriptIds.isEmpty());
assertEquals(repeat, failedCount.get());
service.shutdownNow();
}
private void runScript(CountDownLatch startLatch, CountDownLatch finishLatch, AtomicInteger failedCount,
Map<UUID, Object> scriptIds, String code) {
try {
for (int k = 0; k < 10; k++) {
startLatch.countDown();
startLatch.await();
UUID scriptId = jsSandboxService.eval(JsScriptType.RULE_NODE_SCRIPT, code).get();
scriptIds.put(scriptId, new Object());
jsSandboxService.invokeFunction(scriptId, "{}", "{}", "TEXT").get();
jsSandboxService.release(scriptId).get();
}
} catch (Throwable th) {
failedCount.incrementAndGet();
} finally {
finishLatch.countDown();
}
}
}

57
application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java

@ -1,57 +0,0 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.script;
public class TestNashornJsInvokeService extends AbstractNashornJsInvokeService {
private boolean useJsSandbox;
private final int monitorThreadPoolSize;
private final long maxCpuTime;
private final int maxErrors;
public TestNashornJsInvokeService(boolean useJsSandbox, int monitorThreadPoolSize, long maxCpuTime, int maxErrors) {
this.useJsSandbox = useJsSandbox;
this.monitorThreadPoolSize = monitorThreadPoolSize;
this.maxCpuTime = maxCpuTime;
this.maxErrors = maxErrors;
init();
}
@Override
protected boolean useJsSandbox() {
return useJsSandbox;
}
@Override
protected int getMonitorThreadPoolSize() {
return monitorThreadPoolSize;
}
@Override
protected long getMaxCpuTime() {
return maxCpuTime;
}
@Override
protected int getMaxErrors() {
return maxErrors;
}
@Override
protected long getMaxBlacklistDuration() {
return 100000;
}
}

2
common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateService.java

@ -31,4 +31,6 @@ public interface RuleNodeStateService {
RuleNodeState save(TenantId tenantId, RuleNodeState ruleNodeState); RuleNodeState save(TenantId tenantId, RuleNodeState ruleNodeState);
void removeByRuleNodeId(TenantId tenantId, RuleNodeId selfId); void removeByRuleNodeId(TenantId tenantId, RuleNodeId selfId);
void removeByRuleNodeIdAndEntityId(TenantId tenantId, RuleNodeId selfId, EntityId entityId);
} }

4
common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java

@ -36,9 +36,9 @@ public interface TimeseriesService {
ListenableFuture<List<TsKvEntry>> findAllLatest(TenantId tenantId, EntityId entityId); ListenableFuture<List<TsKvEntry>> findAllLatest(TenantId tenantId, EntityId entityId);
ListenableFuture<List<Void>> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); ListenableFuture<Integer> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry);
ListenableFuture<List<Void>> save(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntry, long ttl); ListenableFuture<Integer> save(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntry, long ttl);
ListenableFuture<List<Void>> saveLatest(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntry); ListenableFuture<List<Void>> saveLatest(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntry);

33
common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java

@ -0,0 +1,33 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.usagerecord;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.id.ApiUsageStateId;
import org.thingsboard.server.common.data.id.TenantId;
public interface ApiUsageStateService {
ApiUsageState createDefaultApiUsageState(TenantId id);
ApiUsageState update(ApiUsageState apiUsageState);
ApiUsageState findTenantApiUsageState(TenantId tenantId);
void deleteApiUsageStateByTenantId(TenantId tenantId);
ApiUsageState findApiUsageStateById(TenantId tenantId, ApiUsageStateId id);
}

38
common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java

@ -0,0 +1,38 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data;
import lombok.Getter;
public enum ApiUsageRecordKey {
TRANSPORT_MSG_COUNT("transportMsgCount", "transportMsgLimit"),
TRANSPORT_DP_COUNT("transportDataPointsCount", "transportDataPointsLimit"),
STORAGE_DP_COUNT("storageDataPointsCount", "storageDataPointsLimit"),
RE_EXEC_COUNT("ruleEngineExecutionCount", "ruleEngineExecutionLimit"),
JS_EXEC_COUNT("jsExecutionCount", "jsExecutionLimit");
@Getter
private final String apiCountKey;
@Getter
private final String apiLimitKey;
ApiUsageRecordKey(String apiCountKey, String apiLimitKey) {
this.apiCountKey = apiCountKey;
this.apiLimitKey = apiLimitKey;
}
}

68
common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java

@ -0,0 +1,68 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.ApiUsageStateId;
@ToString
@EqualsAndHashCode(callSuper = true)
public class ApiUsageState extends BaseData<ApiUsageStateId> implements HasTenantId {
private static final long serialVersionUID = 8250339805336035966L;
@Getter
@Setter
private TenantId tenantId;
@Getter
@Setter
private EntityId entityId;
@Getter
@Setter
private boolean transportEnabled = true;
@Getter
@Setter
private boolean dbStorageEnabled = true;
@Getter
@Setter
private boolean reExecEnabled = true;
@Getter
@Setter
private boolean jsExecEnabled = true;
public ApiUsageState() {
super();
}
public ApiUsageState(ApiUsageStateId id) {
super(id);
}
public ApiUsageState(ApiUsageState ur) {
super(ur);
this.tenantId = ur.getTenantId();
this.entityId = ur.getEntityId();
this.transportEnabled = ur.isTransportEnabled();
this.dbStorageEnabled = ur.isDbStorageEnabled();
this.reExecEnabled = ur.isReExecEnabled();
this.jsExecEnabled = ur.isJsExecEnabled();
}
}

2
common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java

@ -19,5 +19,5 @@ package org.thingsboard.server.common.data;
* @author Andrew Shvayka * @author Andrew Shvayka
*/ */
public enum EntityType { public enum EntityType {
TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, RULE_CHAIN, RULE_NODE, ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, TENANT_PROFILE, DEVICE_PROFILE TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, RULE_CHAIN, RULE_NODE, ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, TENANT_PROFILE, DEVICE_PROFILE, API_USAGE_STATE;
} }

12
common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java

@ -21,6 +21,8 @@ import lombok.Data;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.id.TenantProfileId;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
@ -78,15 +80,21 @@ public class TenantProfile extends SearchTextBased<TenantProfileId> implements H
profileData = mapper.readValue(new ByteArrayInputStream(profileDataBytes), TenantProfileData.class); profileData = mapper.readValue(new ByteArrayInputStream(profileDataBytes), TenantProfileData.class);
} catch (IOException e) { } catch (IOException e) {
log.warn("Can't deserialize tenant profile data: ", e); log.warn("Can't deserialize tenant profile data: ", e);
return null; return createDefaultTenantProfileData();
} }
return profileData; return profileData;
} else { } else {
return null; return createDefaultTenantProfileData();
} }
} }
} }
public TenantProfileData createDefaultTenantProfileData() {
TenantProfileData tpd = new TenantProfileData();
tpd.setConfiguration(new DefaultTenantProfileConfiguration());
return tpd;
}
public void setProfileData(TenantProfileData data) { public void setProfileData(TenantProfileData data) {
this.profileData = data; this.profileData = data;
try { try {

26
common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileData.java → common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileType.java

@ -15,28 +15,6 @@
*/ */
package org.thingsboard.server.common.data; package org.thingsboard.server.common.data;
import com.fasterxml.jackson.annotation.JsonAnyGetter; public enum TenantProfileType {
import com.fasterxml.jackson.annotation.JsonAnySetter; DEFAULT
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Data;
import java.util.HashMap;
import java.util.Map;
@Data
public class TenantProfileData {
@JsonIgnore
private Map<String, Object> properties = new HashMap<>();
@JsonAnyGetter
public Map<String, Object> properties() {
return this.properties;
}
@JsonAnySetter
public void put(String name, Object value) {
this.properties.put(name, value);
}
} }

3
common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java

@ -18,6 +18,7 @@ package org.thingsboard.server.common.data.device.profile;
import lombok.Data; import lombok.Data;
import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmSeverity;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -27,7 +28,7 @@ public class DeviceProfileAlarm {
private String id; private String id;
private String alarmType; private String alarmType;
private Map<AlarmSeverity, AlarmRule> createRules; private LinkedHashMap<AlarmSeverity, AlarmRule> createRules;
private AlarmRule clearRule; private AlarmRule clearRule;
// Hidden in advanced settings // Hidden in advanced settings

42
common/data/src/main/java/org/thingsboard/server/common/data/id/ApiUsageStateId.java

@ -0,0 +1,42 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.id;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.thingsboard.server.common.data.EntityType;
import java.util.UUID;
public class ApiUsageStateId extends UUIDBased implements EntityId {
@JsonCreator
public ApiUsageStateId(@JsonProperty("id") UUID id) {
super(id);
}
public static ApiUsageStateId fromString(String userId) {
return new ApiUsageStateId(UUID.fromString(userId));
}
@JsonIgnore
@Override
public EntityType getEntityType() {
return EntityType.API_USAGE_STATE;
}
}

2
common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java

@ -66,6 +66,8 @@ public class EntityIdFactory {
return new DeviceProfileId(uuid); return new DeviceProfileId(uuid);
case TENANT_PROFILE: case TENANT_PROFILE:
return new TenantProfileId(uuid); return new TenantProfileId(uuid);
case API_USAGE_STATE:
return new ApiUsageStateId(uuid);
} }
throw new IllegalArgumentException("EntityType " + type + " is not supported!"); throw new IllegalArgumentException("EntityType " + type + " is not supported!");
} }

19
common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java

@ -19,7 +19,7 @@ import java.util.Objects;
import java.util.Optional; import java.util.Optional;
public class BasicTsKvEntry implements TsKvEntry { public class BasicTsKvEntry implements TsKvEntry {
private static final int MAX_CHARS_PER_DATA_POINT = 512;
private final long ts; private final long ts;
private final KvEntry kv; private final KvEntry kv;
@ -99,4 +99,21 @@ public class BasicTsKvEntry implements TsKvEntry {
public String getValueAsString() { public String getValueAsString() {
return kv.getValueAsString(); return kv.getValueAsString();
} }
@Override
public int getDataPoints() {
int length;
switch (getDataType()) {
case STRING:
length = getStrValue().get().length();
break;
case JSON:
length = getJsonValue().get().length();
break;
default:
return 1;
}
return Math.max(1, (length + MAX_CHARS_PER_DATA_POINT - 1) / MAX_CHARS_PER_DATA_POINT);
}
} }

5
common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java

@ -15,6 +15,8 @@
*/ */
package org.thingsboard.server.common.data.kv; package org.thingsboard.server.common.data.kv;
import com.fasterxml.jackson.annotation.JsonIgnore;
/** /**
* Represents time series KV data entry * Represents time series KV data entry
* *
@ -25,4 +27,7 @@ public interface TsKvEntry extends KvEntry {
long getTs(); long getTs();
@JsonIgnore
int getDataPoints();
} }

2
common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java

@ -21,5 +21,5 @@ import java.io.Serializable;
* @author Andrew Shvayka * @author Andrew Shvayka
*/ */
public enum ComponentLifecycleEvent implements Serializable { public enum ComponentLifecycleEvent implements Serializable {
CREATED, STARTED, ACTIVATED, SUSPENDED, UPDATED, STOPPED, DELETED CREATED, STARTED, ACTIVATED, SUSPENDED, UPDATED, STOPPED, DELETED, ADDED_TO_ALLOW_LIST, ADDED_TO_DENY_LIST
} }

28
common/data/src/main/java/org/thingsboard/server/common/data/query/ApiUsageStateFilter.java

@ -0,0 +1,28 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.query;
import lombok.Data;
import org.thingsboard.server.common.data.id.EntityId;
@Data
public class ApiUsageStateFilter implements EntityFilter {
@Override
public EntityFilterType getType() {
return EntityFilterType.API_USAGE_STATE;
}
}

1
common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java

@ -32,6 +32,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
@JsonSubTypes.Type(value = AssetTypeFilter.class, name = "assetType"), @JsonSubTypes.Type(value = AssetTypeFilter.class, name = "assetType"),
@JsonSubTypes.Type(value = DeviceTypeFilter.class, name = "deviceType"), @JsonSubTypes.Type(value = DeviceTypeFilter.class, name = "deviceType"),
@JsonSubTypes.Type(value = EntityViewTypeFilter.class, name = "entityViewType"), @JsonSubTypes.Type(value = EntityViewTypeFilter.class, name = "entityViewType"),
@JsonSubTypes.Type(value = ApiUsageStateFilter.class, name = "apiUsageState"),
@JsonSubTypes.Type(value = RelationsQueryFilter.class, name = "relationsQuery"), @JsonSubTypes.Type(value = RelationsQueryFilter.class, name = "relationsQuery"),
@JsonSubTypes.Type(value = AssetSearchQueryFilter.class, name = "assetSearchQuery"), @JsonSubTypes.Type(value = AssetSearchQueryFilter.class, name = "assetSearchQuery"),
@JsonSubTypes.Type(value = DeviceSearchQueryFilter.class, name = "deviceSearchQuery"), @JsonSubTypes.Type(value = DeviceSearchQueryFilter.class, name = "deviceSearchQuery"),

3
common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilterType.java

@ -25,7 +25,8 @@ public enum EntityFilterType {
RELATIONS_QUERY("relationsQuery"), RELATIONS_QUERY("relationsQuery"),
ASSET_SEARCH_QUERY("assetSearchQuery"), ASSET_SEARCH_QUERY("assetSearchQuery"),
DEVICE_SEARCH_QUERY("deviceSearchQuery"), DEVICE_SEARCH_QUERY("deviceSearchQuery"),
ENTITY_VIEW_SEARCH_QUERY("entityViewSearchQuery"); ENTITY_VIEW_SEARCH_QUERY("entityViewSearchQuery"),
API_USAGE_STATE("apiUsageState");
private final String label; private final String label;

69
common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java

@ -0,0 +1,69 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.tenant.profile;
import lombok.Data;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.TenantProfileType;
@Data
public class DefaultTenantProfileConfiguration implements TenantProfileConfiguration {
private long maxDevices;
private long maxAssets;
private String transportTenantMsgRateLimit;
private String transportTenantTelemetryMsgRateLimit;
private String transportTenantTelemetryDataPointsRateLimit;
private String transportDeviceMsgRateLimit;
private String transportDeviceTelemetryMsgRateLimit;
private String transportDeviceTelemetryDataPointsRateLimit;
private long maxTransportMessages;
private long maxTransportDataPoints;
private long maxREExecutions;
private long maxJSExecutions;
private long maxDPStorageDays;
private int maxRuleNodeExecutionsPerMessage;
@Override
public long getProfileThreshold(ApiUsageRecordKey key) {
switch (key) {
case TRANSPORT_MSG_COUNT:
return maxTransportMessages;
case TRANSPORT_DP_COUNT:
return maxTransportDataPoints;
case JS_EXEC_COUNT:
return maxJSExecutions;
case RE_EXEC_COUNT:
return maxREExecutions;
case STORAGE_DP_COUNT:
return maxDPStorageDays;
}
return 0L;
}
@Override
public TenantProfileType getType() {
return TenantProfileType.DEFAULT;
}
@Override
public int getMaxRuleNodeExecsPerMessage() {
return maxRuleNodeExecutionsPerMessage;
}
}

43
common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java

@ -0,0 +1,43 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.tenant.profile;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.TenantProfileType;
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
include = JsonTypeInfo.As.PROPERTY,
property = "type")
@JsonSubTypes({
@JsonSubTypes.Type(value = DefaultTenantProfileConfiguration.class, name = "DEFAULT")})
public interface TenantProfileConfiguration {
@JsonIgnore
TenantProfileType getType();
@JsonIgnore
long getProfileThreshold(ApiUsageRecordKey key);
@JsonIgnore
int getMaxRuleNodeExecsPerMessage();
}

9
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitFactory.java → common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java

@ -13,12 +13,13 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.thingsboard.server.common.transport.limits; package org.thingsboard.server.common.data.tenant.profile;
public interface TransportRateLimitFactory { import lombok.Data;
TransportRateLimit create(TransportRateLimitType type, Object config); @Data
public class TenantProfileData {
TransportRateLimit createDefault(TransportRateLimitType type); private TenantProfileConfiguration configuration;
} }

66
common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java

@ -18,27 +18,27 @@ package org.thingsboard.server.common.msg;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.protobuf.ByteString; import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.InvalidProtocolBufferException;
import lombok.AccessLevel;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.gen.MsgProtos;
import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
import org.thingsboard.server.common.msg.queue.ServiceQueue; import org.thingsboard.server.common.msg.queue.ServiceQueue;
import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import java.io.IOException;
import java.io.Serializable; import java.io.Serializable;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
/** /**
* Created by ashvayka on 13.01.18. * Created by ashvayka on 13.01.18.
*/ */
@Data @Data
@Builder
@Slf4j @Slf4j
public final class TbMsg implements Serializable { public final class TbMsg implements Serializable {
@ -52,61 +52,73 @@ public final class TbMsg implements Serializable {
private final String data; private final String data;
private final RuleChainId ruleChainId; private final RuleChainId ruleChainId;
private final RuleNodeId ruleNodeId; private final RuleNodeId ruleNodeId;
@Getter(value = AccessLevel.NONE)
private final AtomicInteger ruleNodeExecCounter;
public int getAndIncrementRuleNodeCounter() {
return ruleNodeExecCounter.getAndIncrement();
}
//This field is not serialized because we use queues and there is no need to do it //This field is not serialized because we use queues and there is no need to do it
@JsonIgnore @JsonIgnore
transient private final TbMsgCallback callback; transient private final TbMsgCallback callback;
public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator,
metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, 0, TbMsgCallback.EMPTY);
} }
public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) {
return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, TbMsgCallback.EMPTY);
} }
// REALLY NEW MSG
public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data) { public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data) {
return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, TbMsgCallback.EMPTY);
} }
public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data) { public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data) {
return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, null, null, TbMsgCallback.EMPTY); return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, null, null, 0, TbMsgCallback.EMPTY);
} }
// For Tests only
public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, 0, TbMsgCallback.EMPTY);
} }
public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, TbMsgCallback callback) { public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, TbMsgCallback callback) {
return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, callback); return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, callback);
} }
public static TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { public static TbMsg transformMsg(TbMsg tbMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) {
return new TbMsg(origMsg.getQueueName(), origMsg.getId(), origMsg.getTs(), type, originator, metaData.copy(), origMsg.getDataType(), return new TbMsg(tbMsg.getQueueName(), tbMsg.getId(), tbMsg.getTs(), type, originator, metaData.copy(), tbMsg.getDataType(),
data, origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); data, tbMsg.getRuleChainId(), tbMsg.getRuleNodeId(), tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback());
} }
public static TbMsg transformMsg(TbMsg origMsg, RuleChainId ruleChainId) { public static TbMsg transformMsg(TbMsg tbMsg, RuleChainId ruleChainId) {
return new TbMsg(origMsg.queueName, origMsg.id, origMsg.ts, origMsg.type, origMsg.originator, origMsg.metaData, origMsg.dataType, return new TbMsg(tbMsg.queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType,
origMsg.data, ruleChainId, null, origMsg.getCallback()); tbMsg.data, ruleChainId, null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback());
} }
public static TbMsg transformMsg(TbMsg origMsg, String queueName) { public static TbMsg transformMsg(TbMsg tbMsg, String queueName) {
return new TbMsg(queueName, origMsg.id, origMsg.ts, origMsg.type, origMsg.originator, origMsg.metaData, origMsg.dataType, return new TbMsg(queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType,
origMsg.data, origMsg.getRuleChainId(), null, origMsg.getCallback()); tbMsg.data, tbMsg.getRuleChainId(), null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback());
} }
public static TbMsg transformMsg(TbMsg origMsg, RuleChainId ruleChainId, String queueName) { public static TbMsg transformMsg(TbMsg tbMsg, RuleChainId ruleChainId, String queueName) {
return new TbMsg(queueName, origMsg.id, origMsg.ts, origMsg.type, origMsg.originator, origMsg.metaData, origMsg.dataType, return new TbMsg(queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType,
origMsg.data, ruleChainId, null, origMsg.getCallback()); tbMsg.data, ruleChainId, null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback());
} }
public static TbMsg newMsg(TbMsg tbMsg, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { public static TbMsg newMsg(TbMsg tbMsg, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
return new TbMsg(tbMsg.getQueueName(), UUID.randomUUID(), tbMsg.getTs(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(), return new TbMsg(tbMsg.getQueueName(), UUID.randomUUID(), tbMsg.getTs(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(),
tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, tbMsg.ruleNodeExecCounter.get(), TbMsgCallback.EMPTY);
} }
private TbMsg(String queueName, UUID id, long ts, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, private TbMsg(String queueName, UUID id, long ts, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data,
RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { RuleChainId ruleChainId, RuleNodeId ruleNodeId, int ruleNodeExecCounter, TbMsgCallback callback) {
this.id = id; this.id = id;
this.queueName = queueName; this.queueName = queueName;
if (ts > 0) { if (ts > 0) {
@ -121,6 +133,7 @@ public final class TbMsg implements Serializable {
this.data = data; this.data = data;
this.ruleChainId = ruleChainId; this.ruleChainId = ruleChainId;
this.ruleNodeId = ruleNodeId; this.ruleNodeId = ruleNodeId;
this.ruleNodeExecCounter = new AtomicInteger(ruleNodeExecCounter);
if (callback != null) { if (callback != null) {
this.callback = callback; this.callback = callback;
} else { } else {
@ -157,6 +170,7 @@ public final class TbMsg implements Serializable {
builder.setDataType(msg.getDataType().ordinal()); builder.setDataType(msg.getDataType().ordinal());
builder.setData(msg.getData()); builder.setData(msg.getData());
builder.setRuleNodeExecCounter(msg.ruleNodeExecCounter.get());
return builder.build().toByteArray(); return builder.build().toByteArray();
} }
@ -174,18 +188,18 @@ public final class TbMsg implements Serializable {
ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB())); ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB()));
} }
TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()];
return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, callback); return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, proto.getRuleNodeExecCounter(), callback);
} catch (InvalidProtocolBufferException e) { } catch (InvalidProtocolBufferException e) {
throw new IllegalStateException("Could not parse protobuf for TbMsg", e); throw new IllegalStateException("Could not parse protobuf for TbMsg", e);
} }
} }
public TbMsg copyWithRuleChainId(RuleChainId ruleChainId) { public TbMsg copyWithRuleChainId(RuleChainId ruleChainId) {
return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, callback); return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, this.ruleNodeExecCounter.get(), callback);
} }
public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) { public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, callback); return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, this.ruleNodeExecCounter.get(), callback);
} }
public TbMsgCallback getCallback() { public TbMsgCallback getCallback() {

4
common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java

@ -50,6 +50,10 @@ public class TopicPartitionInfo {
this.fullTopicName = tmp; this.fullTopicName = tmp;
} }
public TopicPartitionInfo newByTopic(String topic) {
return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.myPartition);
}
public String getTopic() { public String getTopic() {
return topic; return topic;
} }

48
common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java

@ -15,16 +15,64 @@
*/ */
package org.thingsboard.server.common.msg.tools; package org.thingsboard.server.common.msg.tools;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZoneId; import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.temporal.ChronoUnit;
import java.time.temporal.TemporalAdjuster;
import java.time.temporal.TemporalAdjusters;
import java.time.temporal.TemporalUnit;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import static java.time.temporal.ChronoField.DAY_OF_MONTH;
import static java.time.temporal.ChronoUnit.MONTHS;
public class SchedulerUtils { public class SchedulerUtils {
private final static ZoneId UTC = ZoneId.of("UTC");
private static final ConcurrentMap<String, ZoneId> tzMap = new ConcurrentHashMap<>(); private static final ConcurrentMap<String, ZoneId> tzMap = new ConcurrentHashMap<>();
public static ZoneId getZoneId(String tz) { public static ZoneId getZoneId(String tz) {
return tzMap.computeIfAbsent(tz == null || tz.isEmpty() ? "UTC" : tz, ZoneId::of); return tzMap.computeIfAbsent(tz == null || tz.isEmpty() ? "UTC" : tz, ZoneId::of);
} }
public static long getStartOfCurrentHour() {
return getStartOfCurrentHour(UTC);
}
public static long getStartOfCurrentHour(ZoneId zoneId) {
return LocalDateTime.now(ZoneOffset.UTC).atZone(zoneId).truncatedTo(ChronoUnit.HOURS).toInstant().toEpochMilli();
}
public static long getStartOfCurrentMonth() {
return getStartOfCurrentMonth(UTC);
}
public static long getStartOfCurrentMonth(ZoneId zoneId) {
return LocalDate.now().withDayOfMonth(1).atStartOfDay(zoneId).toInstant().toEpochMilli();
}
public static long getStartOfNextMonth() {
return getStartOfNextMonth(UTC);
}
public static long getStartOfNextMonth(ZoneId zoneId) {
return LocalDate.now().with(TemporalAdjusters.firstDayOfNextMonth()).atStartOfDay(zoneId).toInstant().toEpochMilli();
}
public static long getStartOfNextNextMonth() {
return getStartOfNextNextMonth(UTC);
}
public static long getStartOfNextNextMonth(ZoneId zoneId) {
return LocalDate.now().with(firstDayOfNextNextMonth()).atStartOfDay(zoneId).toInstant().toEpochMilli();
}
public static TemporalAdjuster firstDayOfNextNextMonth() {
return (temporal) -> temporal.with(DAY_OF_MONTH, 1).plus(2, MONTHS);
}
} }

1
common/message/src/main/proto/tbmsg.proto

@ -45,4 +45,5 @@ message TbMsgProto {
string data = 14; string data = 14;
int64 ts = 15; int64 ts = 15;
int32 ruleNodeExecCounter = 16;
} }

45
common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java

@ -0,0 +1,45 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.common;
import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.queue.TbQueueCallback;
import org.thingsboard.server.queue.TbQueueMsgMetadata;
import java.util.concurrent.atomic.AtomicInteger;
public class MultipleTbQueueCallbackWrapper implements TbQueueCallback {
private final AtomicInteger tbQueueCallbackCount;
private final TbQueueCallback callback;
public MultipleTbQueueCallbackWrapper(int tbQueueCallbackCount, TbQueueCallback callback) {
this.tbQueueCallbackCount = new AtomicInteger(tbQueueCallbackCount);
this.callback = callback;
}
@Override
public void onSuccess(TbQueueMsgMetadata metadata) {
if (tbQueueCallbackCount.decrementAndGet() <= 0) {
callback.onSuccess(metadata);
}
}
@Override
public void onFailure(Throwable t) {
callback.onFailure(new RuleEngineException(t.getMessage()));
}
}

45
common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java

@ -22,7 +22,7 @@ import org.springframework.stereotype.Component;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest;
import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.*;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
@ -91,64 +91,64 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToTransportMsg>> createTransportNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic()); return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> createRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> createTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) {
return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(), return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() {
return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings,
partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> createToCoreMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() {
return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(), return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() {
return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings,
partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>> createTransportApiRequestConsumer() { public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() {
return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(), return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> createTransportApiResponseProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() {
return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getResponsesTopic()); return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getResponsesTopic());
} }
@ -175,6 +175,17 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
return builder.build(); return builder.build();
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic());
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, coreSettings.getUsageStatsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

12
common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java

@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
@ -163,6 +164,17 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory {
return builder.build(); return builder.build();
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic());
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, coreSettings.getUsageStatsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

6
common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java

@ -143,6 +143,11 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory
return builder.build(); return builder.build();
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic());
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {
@ -158,4 +163,5 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory
notificationAdmin.destroy(); notificationAdmin.destroy();
} }
} }
} }

6
common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java

@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
@ -109,6 +110,11 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory {
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic());
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

12
common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java

@ -125,8 +125,20 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
return null; return null;
} }
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
return new InMemoryTbQueueConsumer<>(coreSettings.getUsageStatsTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new InMemoryTbQueueProducer<>(coreSettings.getUsageStatsTopic());
}
@Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}")
private void printInMemoryStats() { private void printInMemoryStats() {
storage.printStats(); storage.printStats();
} }
} }

7
common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java

@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
@ -96,4 +97,10 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory
public TbQueueConsumer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsConsumer() {
return new InMemoryTbQueueConsumer<>(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); return new InMemoryTbQueueConsumer<>(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId());
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new InMemoryTbQueueProducer<>(coreSettings.getUsageStatsTopic());
}
} }

23
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java

@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
@ -249,6 +250,28 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
return builder.build(); return builder.build();
} }
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> consumerBuilder = TbKafkaConsumerTemplate.builder();
consumerBuilder.settings(kafkaSettings);
consumerBuilder.topic(coreSettings.getUsageStatsTopic());
consumerBuilder.clientId("monolith-us-consumer-" + serviceInfoProvider.getServiceId());
consumerBuilder.groupId("monolith-us-consumer-" + serviceInfoProvider.getServiceId());
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(coreAdmin);
return consumerBuilder.build();
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> requestBuilder = TbKafkaProducerTemplate.builder();
requestBuilder.settings(kafkaSettings);
requestBuilder.clientId("monolith-us-producer-" + serviceInfoProvider.getServiceId());
requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic());
requestBuilder.admin(coreAdmin);
return requestBuilder.build();
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

23
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java

@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
@ -219,6 +220,28 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
return builder.build(); return builder.build();
} }
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> consumerBuilder = TbKafkaConsumerTemplate.builder();
consumerBuilder.settings(kafkaSettings);
consumerBuilder.topic(coreSettings.getUsageStatsTopic());
consumerBuilder.clientId("tb-core-us-consumer-" + serviceInfoProvider.getServiceId());
consumerBuilder.groupId("tb-core-us-consumer-" + serviceInfoProvider.getServiceId());
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(coreAdmin);
return consumerBuilder.build();
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> requestBuilder = TbKafkaProducerTemplate.builder();
requestBuilder.settings(kafkaSettings);
requestBuilder.clientId("tb-core-us-producer-" + serviceInfoProvider.getServiceId());
requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic());
requestBuilder.admin(coreAdmin);
return requestBuilder.build();
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

11
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java

@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
@ -192,6 +193,16 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
return builder.build(); return builder.build();
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> requestBuilder = TbKafkaProducerTemplate.builder();
requestBuilder.settings(kafkaSettings);
requestBuilder.clientId("tb-rule-engine-us-producer-" + serviceInfoProvider.getServiceId());
requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic());
requestBuilder.admin(coreAdmin);
return requestBuilder.build();
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

11
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java

@ -21,6 +21,7 @@ import org.springframework.stereotype.Component;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
@ -138,6 +139,16 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory {
return responseBuilder.build(); return responseBuilder.build();
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> requestBuilder = TbKafkaProducerTemplate.builder();
requestBuilder.settings(kafkaSettings);
requestBuilder.clientId("transport-node-us-producer-" + serviceInfoProvider.getServiceId());
requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic());
requestBuilder.admin(coreAdmin);
return requestBuilder.build();
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

12
common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java

@ -27,6 +27,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
@ -183,6 +184,17 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
return builder.build(); return builder.build();
} }
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic());
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

12
common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java

@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
@ -157,6 +158,17 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory {
return builder.build(); return builder.build();
} }
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic());
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

6
common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java

@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
@ -146,6 +147,11 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory
return builder.build(); return builder.build();
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic());
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

6
common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java

@ -21,6 +21,7 @@ import org.springframework.stereotype.Component;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
@ -114,6 +115,11 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory {
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic());
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

52
common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java

@ -22,7 +22,14 @@ import org.springframework.stereotype.Component;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest;
import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
@ -91,64 +98,64 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToTransportMsg>> createTransportNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic()); return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> createRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> createTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) {
return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(), return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() {
return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings,
partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> createToCoreMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() {
return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(), return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() {
return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings,
partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>> createTransportApiRequestConsumer() { public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() {
return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> createTransportApiResponseProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() {
return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getResponsesTopic());
} }
@ -175,6 +182,17 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
return builder.build(); return builder.build();
} }
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic());
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

12
common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java

@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
@ -163,6 +164,17 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory {
return builder.build(); return builder.build();
} }
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic());
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

6
common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java

@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
@ -144,6 +145,11 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor
return builder.build(); return builder.build();
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic());
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

6
common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java

@ -21,6 +21,7 @@ import org.springframework.stereotype.Component;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
@ -111,6 +112,11 @@ public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory {
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic());
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

12
common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java

@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
@ -180,6 +181,17 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul
return builder.build(); return builder.build();
} }
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic());
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

23
common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java

@ -21,10 +21,12 @@ import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
@ -101,7 +103,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory {
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic()); return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic());
} }
@ -111,7 +113,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory {
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
} }
@ -122,10 +124,10 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory {
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() {
return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings,
partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
@ -162,6 +164,17 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory {
return builder.build(); return builder.build();
} }
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic());
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

17
common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java

@ -21,10 +21,12 @@ import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
@ -93,7 +95,7 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic()); return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic());
} }
@ -103,7 +105,7 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
} }
@ -114,10 +116,10 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() {
return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings,
partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
@ -143,6 +145,11 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact
return builder.build(); return builder.build();
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic());
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

36
common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java

@ -18,19 +18,24 @@ package org.thingsboard.server.queue.provider;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings;
import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
@ -69,17 +74,17 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory
} }
@Override @Override
public TbQueueRequestTemplate<TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>, TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> createTransportApiRequestTemplate() { public TbQueueRequestTemplate<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiRequestTemplate() {
TbQueueProducer<TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>> producerTemplate = TbQueueProducer<TbProtoQueueMsg<TransportApiRequestMsg>> producerTemplate =
new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic()); new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic());
TbQueueConsumer<TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> consumerTemplate = TbQueueConsumer<TbProtoQueueMsg<TransportApiResponseMsg>> consumerTemplate =
new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings, new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings,
transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders()));
DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
<TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>, TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> templateBuilder = DefaultTbQueueRequestTemplate.builder(); <TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> templateBuilder = DefaultTbQueueRequestTemplate.builder();
templateBuilder.queueAdmin(transportApiAdmin); templateBuilder.queueAdmin(transportApiAdmin);
templateBuilder.requestTemplate(producerTemplate); templateBuilder.requestTemplate(producerTemplate);
templateBuilder.responseTemplate(consumerTemplate); templateBuilder.responseTemplate(consumerTemplate);
@ -90,20 +95,25 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> createRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic()); return new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> createTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToTransportMsg>> createTransportNotificationsConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsConsumer() {
return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings,
transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic());
} }
@PreDestroy @PreDestroy

10
common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java

@ -21,6 +21,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
@ -33,7 +34,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg;
* Responsible for initialization of various Producers and Consumers used by TB Core Node. * Responsible for initialization of various Producers and Consumers used by TB Core Node.
* Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable
*/ */
public interface TbCoreQueueFactory { public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory {
/** /**
* Used to push messages to instances of TB Transport Service * Used to push messages to instances of TB Transport Service
@ -77,6 +78,13 @@ public interface TbCoreQueueFactory {
*/ */
TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer(); TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer();
/**
* Used to consume messages about usage statistics by TB Core Service
*
* @return
*/
TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer();
/** /**
* Used to consume high priority messages by TB Core Service * Used to consume high priority messages by TB Core Service
* *

34
common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java

@ -17,7 +17,12 @@ package org.thingsboard.server.queue.provider;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -28,11 +33,12 @@ import javax.annotation.PostConstruct;
public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { public class TbCoreQueueProducerProvider implements TbQueueProducerProvider {
private final TbCoreQueueFactory tbQueueProvider; private final TbCoreQueueFactory tbQueueProvider;
private TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToTransportMsg>> toTransport; private TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> toTransport;
private TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> toRuleEngine; private TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> toRuleEngine;
private TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> toTbCore; private TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> toTbCore;
private TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> toRuleEngineNotifications; private TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> toRuleEngineNotifications;
private TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> toTbCoreNotifications; private TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> toTbCoreNotifications;
private TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> toUsageStats;
public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) {
this.tbQueueProvider = tbQueueProvider; this.tbQueueProvider = tbQueueProvider;
@ -45,30 +51,36 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider {
this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer();
this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer();
this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer();
this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer();
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToTransportMsg>> getTransportNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> getTransportNotificationsMsgProducer() {
return toTransport; return toTransport;
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> getRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> getRuleEngineMsgProducer() {
return toRuleEngine; return toRuleEngine;
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> getRuleEngineNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> getRuleEngineNotificationsMsgProducer() {
return toRuleEngineNotifications; return toRuleEngineNotifications;
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> getTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> getTbCoreMsgProducer() {
return toTbCore; return toTbCore;
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> getTbCoreNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> getTbCoreNotificationsMsgProducer() {
return toTbCoreNotifications; return toTbCoreNotifications;
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> getTbUsageStatsMsgProducer() {
return toUsageStats;
}
} }

7
common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java

@ -20,6 +20,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -63,4 +64,10 @@ public interface TbQueueProducerProvider {
*/ */
TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> getTbCoreNotificationsMsgProducer(); TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> getTbCoreNotificationsMsgProducer();
/**
* Used to push messages to other instances of TB Core Service
*
* @return
*/
TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> getTbUsageStatsMsgProducer();
} }

35
common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java

@ -17,7 +17,12 @@ package org.thingsboard.server.queue.provider;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -28,12 +33,12 @@ import javax.annotation.PostConstruct;
public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { public class TbRuleEngineProducerProvider implements TbQueueProducerProvider {
private final TbRuleEngineQueueFactory tbQueueProvider; private final TbRuleEngineQueueFactory tbQueueProvider;
private TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToTransportMsg>> toTransport; private TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> toTransport;
private TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> toRuleEngine; private TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> toRuleEngine;
private TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> toTbCore; private TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> toTbCore;
private TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> toRuleEngineNotifications; private TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> toRuleEngineNotifications;
private TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> toTbCoreNotifications; private TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> toTbCoreNotifications;
private TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> toUsageStats;
public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) {
this.tbQueueProvider = tbQueueProvider; this.tbQueueProvider = tbQueueProvider;
@ -46,30 +51,36 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider {
this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer();
this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer();
this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer();
this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer();
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToTransportMsg>> getTransportNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> getTransportNotificationsMsgProducer() {
return toTransport; return toTransport;
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> getRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> getRuleEngineMsgProducer() {
return toRuleEngine; return toRuleEngine;
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> getRuleEngineNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> getRuleEngineNotificationsMsgProducer() {
return toRuleEngineNotifications; return toRuleEngineNotifications;
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> getTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> getTbCoreMsgProducer() {
return toTbCore; return toTbCore;
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> getTbCoreNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> getTbCoreNotificationsMsgProducer() {
return toTbCoreNotifications; return toTbCoreNotifications;
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> getTbUsageStatsMsgProducer() {
return toUsageStats;
}
} }

2
common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java

@ -32,7 +32,7 @@ import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
* Responsible for initialization of various Producers and Consumers used by TB Core Node. * Responsible for initialization of various Producers and Consumers used by TB Core Node.
* Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable
*/ */
public interface TbRuleEngineQueueFactory { public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory {
/** /**
* Used to push messages to instances of TB Transport Service * Used to push messages to instances of TB Transport Service

10
common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java

@ -15,17 +15,17 @@
*/ */
package org.thingsboard.server.queue.provider; package org.thingsboard.server.queue.provider;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
public interface TbTransportQueueFactory { public interface TbTransportQueueFactory extends TbUsageStatsClientQueueFactory {
TbQueueRequestTemplate<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiRequestTemplate(); TbQueueRequestTemplate<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiRequestTemplate();

28
common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java

@ -17,7 +17,12 @@ package org.thingsboard.server.queue.provider;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -28,8 +33,9 @@ import javax.annotation.PostConstruct;
public class TbTransportQueueProducerProvider implements TbQueueProducerProvider { public class TbTransportQueueProducerProvider implements TbQueueProducerProvider {
private final TbTransportQueueFactory tbQueueProvider; private final TbTransportQueueFactory tbQueueProvider;
private TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> toRuleEngine; private TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> toRuleEngine;
private TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> toTbCore; private TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> toTbCore;
private TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> toUsageStats;
public TbTransportQueueProducerProvider(TbTransportQueueFactory tbQueueProvider) { public TbTransportQueueProducerProvider(TbTransportQueueFactory tbQueueProvider) {
this.tbQueueProvider = tbQueueProvider; this.tbQueueProvider = tbQueueProvider;
@ -39,30 +45,36 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider
public void init() { public void init() {
this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); this.toTbCore = tbQueueProvider.createTbCoreMsgProducer();
this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer();
this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer();
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToTransportMsg>> getTransportNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> getTransportNotificationsMsgProducer() {
throw new RuntimeException("Not Implemented! Should not be used by Transport!"); throw new RuntimeException("Not Implemented! Should not be used by Transport!");
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> getRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> getRuleEngineMsgProducer() {
return toRuleEngine; return toRuleEngine;
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> getTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> getTbCoreMsgProducer() {
return toTbCore; return toTbCore;
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> getRuleEngineNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> getRuleEngineNotificationsMsgProducer() {
throw new RuntimeException("Not Implemented! Should not be used by Transport!"); throw new RuntimeException("Not Implemented! Should not be used by Transport!");
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> getTbCoreNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> getTbCoreNotificationsMsgProducer() {
throw new RuntimeException("Not Implemented! Should not be used by Transport!"); throw new RuntimeException("Not Implemented! Should not be used by Transport!");
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> getTbUsageStatsMsgProducer() {
return toUsageStats;
}
} }

26
common/queue/src/main/java/org/thingsboard/server/queue/provider/TbUsageStatsClientQueueFactory.java

@ -0,0 +1,26 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.provider;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
public interface TbUsageStatsClientQueueFactory {
TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer();
}

61
common/queue/src/main/java/org/thingsboard/server/queue/scheduler/DefaultSchedulerComponent.java

@ -0,0 +1,61 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.scheduler;
import org.springframework.stereotype.Component;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.concurrent.Callable;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
@Component
public class DefaultSchedulerComponent implements SchedulerComponent{
protected ScheduledExecutorService schedulerExecutor;
@PostConstruct
public void init(){
this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("queue-scheduler"));
}
@PreDestroy
public void destroy() {
if (schedulerExecutor != null) {
schedulerExecutor.shutdownNow();
}
}
public ScheduledFuture<?> schedule(Runnable command, long delay, TimeUnit unit) {
return schedulerExecutor.schedule(command, delay, unit);
}
public <V> ScheduledFuture<V> schedule(Callable<V> callable, long delay, TimeUnit unit) {
return schedulerExecutor.schedule(callable, delay, unit);
}
public ScheduledFuture<?> scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit) {
return schedulerExecutor.scheduleAtFixedRate(command, initialDelay, period, unit);
}
public ScheduledFuture<?> scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit) {
return schedulerExecutor.scheduleWithFixedDelay(command, initialDelay, delay, unit);
}
}

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save