Browse Source

merge with master

pull/3740/head
ShvaykaD 6 years ago
parent
commit
18a57d638d
  1. 8
      application/src/main/data/json/system/widget_bundles/input_widgets.json
  2. 1
      application/src/main/data/upgrade/3.1.1/schema_update_before.sql
  3. 13
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  4. 12
      application/src/main/java/org/thingsboard/server/actors/app/AppActor.java
  5. 32
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
  6. 7
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java
  7. 6
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java
  8. 51
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java
  9. 2
      application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java
  10. 6
      application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java
  11. 63
      application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java
  12. 2
      application/src/main/java/org/thingsboard/server/controller/AlarmController.java
  13. 7
      application/src/main/java/org/thingsboard/server/controller/AuthController.java
  14. 3
      application/src/main/java/org/thingsboard/server/controller/BaseController.java
  15. 3
      application/src/main/java/org/thingsboard/server/controller/DeviceController.java
  16. 4
      application/src/main/java/org/thingsboard/server/controller/RuleChainController.java
  17. 5
      application/src/main/java/org/thingsboard/server/controller/TelemetryController.java
  18. 2
      application/src/main/java/org/thingsboard/server/controller/TenantController.java
  19. 10
      application/src/main/java/org/thingsboard/server/controller/UserController.java
  20. 370
      application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java
  21. 38
      application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java
  22. 186
      application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java
  23. 15
      application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java
  24. 24
      application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java
  25. 56
      application/src/main/java/org/thingsboard/server/service/profile/DefaultTbDeviceProfileCache.java
  26. 4
      application/src/main/java/org/thingsboard/server/service/profile/TbDeviceProfileCache.java
  27. 36
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java
  28. 113
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  29. 48
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
  30. 3
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java
  31. 3
      application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java
  32. 31
      application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java
  33. 64
      application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java
  34. 13
      application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java
  35. 2
      application/src/main/java/org/thingsboard/server/service/script/JsExecutorService.java
  36. 5
      application/src/main/java/org/thingsboard/server/service/script/JsInvokeService.java
  37. 6
      application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java
  38. 6
      application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java
  39. 11
      application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java
  40. 41
      application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java
  41. 2
      application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationFailureHandler.java
  42. 1
      application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java
  43. 3
      application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java
  44. 1
      application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java
  45. 24
      application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java
  46. 5
      application/src/main/java/org/thingsboard/server/service/security/system/SystemSecurityService.java
  47. 8
      application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java
  48. 6
      application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java
  49. 109
      application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java
  50. 46
      application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java
  51. 2
      application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java
  52. 47
      application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java
  53. 2
      application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java
  54. 2
      application/src/main/resources/thingsboard.yml
  55. 7
      application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java
  56. 4
      application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java
  57. 250
      application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java
  58. 57
      application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java
  59. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateService.java
  60. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TbTenantProfileCache.java
  61. 4
      common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java
  62. 33
      common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java
  63. 29
      common/data/src/main/java/org/thingsboard/server/common/data/ApiFeature.java
  64. 60
      common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java
  65. 84
      common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java
  66. 26
      common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageStateValue.java
  67. 2
      common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java
  68. 2
      common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java
  69. 12
      common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java
  70. 20
      common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileType.java
  71. 3
      common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java
  72. 42
      common/data/src/main/java/org/thingsboard/server/common/data/id/ApiUsageStateId.java
  73. 2
      common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java
  74. 19
      common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java
  75. 5
      common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java
  76. 2
      common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java
  77. 28
      common/data/src/main/java/org/thingsboard/server/common/data/query/ApiUsageStateFilter.java
  78. 1
      common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java
  79. 3
      common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilterType.java
  80. 75
      common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java
  81. 46
      common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java
  82. 9
      common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java
  83. 64
      common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java
  84. 4
      common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java
  85. 48
      common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java
  86. 1
      common/message/src/main/proto/tbmsg.proto
  87. 45
      common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java
  88. 2
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java
  89. 2
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java
  90. 45
      common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java
  91. 12
      common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java
  92. 6
      common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java
  93. 6
      common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java
  94. 12
      common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java
  95. 7
      common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java
  96. 23
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java
  97. 23
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java
  98. 11
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java
  99. 11
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java
  100. 12
      common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java

8
application/src/main/data/json/system/widget_bundles/input_widgets.json

@ -391,16 +391,16 @@
}, },
{ {
"alias": "web_camera_input", "alias": "web_camera_input",
"name": "Web Camera Input", "name": "Photo camera input",
"descriptor": { "descriptor": {
"type": "latest", "type": "latest",
"sizeX": 7.5, "sizeX": 7.5,
"sizeY": 3, "sizeY": 3,
"resources": [], "resources": [],
"templateHtml": "<tb-web-camera-widget \n [ctx]=\"ctx\">\n</tb-web-camera-widget>", "templateHtml": "<tb-photo-camera-widget \n [ctx]=\"ctx\">\n</tb-photo-camera-widget>",
"templateCss": "", "templateCss": "",
"controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.webCameraInputWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n maxDataKeys: 1,\n singleEntity: true\n }\n}\n\nself.onDestroy = function() {\n}\n", "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.photoCameraInputWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n maxDataKeys: 1,\n singleEntity: true\n }\n}\n\nself.onDestroy = function() {\n}\n",
"settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"Web Camera\",\n \"properties\": {\n \"widgetTitle\": {\n \"title\": \"Widget title\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"imageFormat\": {\n \"title\": \"Image Format\",\n \"type\": \"string\",\n \"default\": \"image/png\"\n },\n \"imageQuality\":{\n \"title\":\"Image quality that use lossy compression such as jpeg and webp\",\n \"type\":\"number\",\n \"default\": 0.92,\n \"min\": 0,\n \"max\": 1\n },\n \"maxWidth\": {\n \"title\": \"The maximal image width\",\n \"type\": \"number\",\n \"default\": 640\n }, \n \"maxHeight\": {\n \"title\": \"The maximal image heigth\",\n \"type\": \"number\",\n \"default\": 480\n }\n },\n \"required\": []\n },\n \"form\": [\n \"widgetTitle\",\n {\n \"key\": \"imageFormat\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [\n {\n \"value\": \"image/jpeg\",\n \"label\": \"JPEG\"\n },\n {\n \"value\": \"image/png\",\n \"label\": \"PNG\"\n },\n {\n \"value\": \"image/webp\",\n \"label\": \"WEBP\"\n }\n ]\n },\n \"imageQuality\",\n \"maxWidth\",\n \"maxHeight\"\n ]\n}", "settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"Photo Camera\",\n \"properties\": {\n \"widgetTitle\": {\n \"title\": \"Widget title\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"imageFormat\": {\n \"title\": \"Image Format\",\n \"type\": \"string\",\n \"default\": \"image/png\"\n },\n \"imageQuality\":{\n \"title\":\"Image quality that use lossy compression such as jpeg and webp\",\n \"type\":\"number\",\n \"default\": 0.92,\n \"min\": 0,\n \"max\": 1\n },\n \"maxWidth\": {\n \"title\": \"The maximal image width\",\n \"type\": \"number\",\n \"default\": 640\n }, \n \"maxHeight\": {\n \"title\": \"The maximal image heigth\",\n \"type\": \"number\",\n \"default\": 480\n }\n },\n \"required\": []\n },\n \"form\": [\n \"widgetTitle\",\n {\n \"key\": \"imageFormat\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [\n {\n \"value\": \"image/jpeg\",\n \"label\": \"JPEG\"\n },\n {\n \"value\": \"image/png\",\n \"label\": \"PNG\"\n },\n {\n \"value\": \"image/webp\",\n \"label\": \"WEBP\"\n }\n ]\n },\n \"imageQuality\",\n \"maxWidth\",\n \"maxHeight\"\n ]\n}",
"dataKeySettingsSchema": "{}\n", "dataKeySettingsSchema": "{}\n",
"defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Random\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = prevValue + Math.random() * 100 - 50;\\nvar multiplier = Math.pow(10, 2 || 0);\\nvar value = Math.round(value * multiplier) / multiplier;\\nif (value < -1000) {\\n\\tvalue = -1000;\\n} else if (value > 1000) {\\n\\tvalue = 1000;\\n}\\nreturn value;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Web Camera Input\",\"showTitleIcon\":false,\"titleIcon\":\"more_horiz\",\"iconColor\":\"rgba(0, 0, 0, 0.87)\",\"iconSize\":\"24px\",\"titleTooltip\":\"\",\"dropShadow\":true,\"enableFullscreen\":false,\"widgetStyle\":{},\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"useDashboardTimewindow\":true,\"displayTimewindow\":true,\"showLegend\":false,\"actions\":{}}" "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Random\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = prevValue + Math.random() * 100 - 50;\\nvar multiplier = Math.pow(10, 2 || 0);\\nvar value = Math.round(value * multiplier) / multiplier;\\nif (value < -1000) {\\n\\tvalue = -1000;\\n} else if (value > 1000) {\\n\\tvalue = 1000;\\n}\\nreturn value;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Web Camera Input\",\"showTitleIcon\":false,\"titleIcon\":\"more_horiz\",\"iconColor\":\"rgba(0, 0, 0, 0.87)\",\"iconSize\":\"24px\",\"titleTooltip\":\"\",\"dropShadow\":true,\"enableFullscreen\":false,\"widgetStyle\":{},\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"useDashboardTimewindow\":true,\"displayTimewindow\":true,\"showLegend\":false,\"actions\":{}}"
} }

1
application/src/main/data/upgrade/3.1.1/schema_update_before.sql

@ -96,6 +96,7 @@ CREATE TABLE IF NOT EXISTS device_profile (
is_default boolean, is_default boolean,
tenant_id uuid, tenant_id uuid,
default_rule_chain_id uuid, default_rule_chain_id uuid,
default_queue_name varchar(255),
provision_device_key varchar, provision_device_key varchar,
CONSTRAINT device_profile_name_unq_key UNIQUE (tenant_id, name), CONSTRAINT device_profile_name_unq_key UNIQUE (tenant_id, name),
CONSTRAINT device_provision_key_unq_key UNIQUE (provision_device_key), CONSTRAINT device_provision_key_unq_key UNIQUE (provision_device_key),

13
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -32,7 +32,6 @@ import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache;
import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.service.ActorService;
import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.actors.tenant.DebugTbRateLimits;
import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.DataConstants;
@ -65,6 +64,8 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.DbCallbackExecutorService;
@ -72,7 +73,7 @@ import org.thingsboard.server.service.executors.ExternalCallExecutorService;
import org.thingsboard.server.service.executors.SharedEventLoopGroupService; import org.thingsboard.server.service.executors.SharedEventLoopGroupService;
import org.thingsboard.server.service.mail.MailExecutorService; import org.thingsboard.server.service.mail.MailExecutorService;
import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.service.profile.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.queue.TbClusterService;
import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService;
import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService;
@ -106,6 +107,14 @@ public class ActorSystemContext {
return debugPerTenantLimits; return debugPerTenantLimits;
} }
@Autowired
@Getter
private TbApiUsageStateService apiUsageStateService;
@Autowired
@Getter
private TbApiUsageClient apiUsageClient;
@Autowired @Autowired
@Getter @Getter
@Setter @Setter

12
application/src/main/java/org/thingsboard/server/actors/app/AppActor.java

@ -30,7 +30,6 @@ import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.TenantProfileId;
import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.page.PageDataIterable;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.MsgType;
@ -40,10 +39,8 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg;
import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.service.profile.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper;
import java.util.HashSet; import java.util.HashSet;
@ -150,15 +147,12 @@ public class AppActor extends ContextAwareActor {
private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) {
TbActorRef target = null; TbActorRef target = null;
if (TenantId.SYS_TENANT_ID.equals(msg.getTenantId())) { if (TenantId.SYS_TENANT_ID.equals(msg.getTenantId())) {
if (msg.getEntityId().getEntityType() == EntityType.TENANT_PROFILE) { if (!EntityType.TENANT_PROFILE.equals(msg.getEntityId().getEntityType())) {
tenantProfileCache.evict(new TenantProfileId(msg.getEntityId().getId()));
} else {
log.warn("Message has system tenant id: {}", msg); log.warn("Message has system tenant id: {}", msg);
} }
} else { } else {
if (msg.getEntityId().getEntityType() == EntityType.TENANT) { if (EntityType.TENANT.equals(msg.getEntityId().getEntityType())) {
TenantId tenantId = new TenantId(msg.getEntityId().getId()); TenantId tenantId = new TenantId(msg.getEntityId().getId());
tenantProfileCache.evict(tenantId);
if (msg.getEvent() == ComponentLifecycleEvent.DELETED) { if (msg.getEvent() == ComponentLifecycleEvent.DELETED) {
log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg);
deletedTenants.add(tenantId); deletedTenants.add(tenantId);

32
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java

@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.Alarm;
import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.RuleNodeId;
@ -73,6 +74,7 @@ import org.thingsboard.server.service.script.RuleNodeJsScriptEngine;
import java.util.Collections; import java.util.Collections;
import java.util.Set; import java.util.Set;
import java.util.function.BiConsumer;
import java.util.function.Consumer; import java.util.function.Consumer;
/** /**
@ -257,26 +259,26 @@ class DefaultTbContext implements TbContext {
} }
public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) { public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) {
return entityCreatedMsg(customer, customer.getId(), ruleNodeId); return entityActionMsg(customer, customer.getId(), ruleNodeId, DataConstants.ENTITY_CREATED);
} }
public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) { public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) {
return entityCreatedMsg(device, device.getId(), ruleNodeId); return entityActionMsg(device, device.getId(), ruleNodeId, DataConstants.ENTITY_CREATED);
} }
public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) { public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) {
return entityCreatedMsg(asset, asset.getId(), ruleNodeId); return entityActionMsg(asset, asset.getId(), ruleNodeId, DataConstants.ENTITY_CREATED);
} }
public TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId) { public TbMsg alarmActionMsg(Alarm alarm, RuleNodeId ruleNodeId, String action) {
return entityCreatedMsg(alarm, alarm.getId(), ruleNodeId); return entityActionMsg(alarm, alarm.getId(), ruleNodeId, action);
} }
public <E, I extends EntityId> TbMsg entityCreatedMsg(E entity, I id, RuleNodeId ruleNodeId) { public <E, I extends EntityId> TbMsg entityActionMsg(E entity, I id, RuleNodeId ruleNodeId, String action) {
try { try {
return TbMsg.newMsg(DataConstants.ENTITY_CREATED, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity))); return TbMsg.newMsg(action, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity)));
} catch (JsonProcessingException | IllegalArgumentException e) { } catch (JsonProcessingException | IllegalArgumentException e) {
throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " created msg: " + e); throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " " + action + " msg: " + e);
} }
} }
@ -312,7 +314,7 @@ class DefaultTbContext implements TbContext {
@Override @Override
public ScriptEngine createJsScriptEngine(String script, String... argNames) { public ScriptEngine createJsScriptEngine(String script, String... argNames) {
return new RuleNodeJsScriptEngine(mainCtx.getJsSandbox(), nodeCtx.getSelf().getId(), script, argNames); return new RuleNodeJsScriptEngine(getTenantId(), mainCtx.getJsSandbox(), nodeCtx.getSelf().getId(), script, argNames);
} }
@Override @Override
@ -479,8 +481,16 @@ class DefaultTbContext implements TbContext {
} }
@Override @Override
public void addProfileListener(Consumer<DeviceProfile> listener) { public void removeRuleNodeStateForEntity(EntityId entityId) {
mainCtx.getDeviceProfileCache().addListener(getTenantId(), getSelfId(), listener); if (log.isDebugEnabled()) {
log.debug("[{}][{}][{}] Remove Rule Node State for entity.", getTenantId(), getSelfId(), entityId);
}
mainCtx.getRuleNodeStateService().removeByRuleNodeIdAndEntityId(getTenantId(), getSelfId(), entityId);
}
@Override
public void addDeviceProfileListeners(Consumer<DeviceProfile> profileListener, BiConsumer<DeviceId, DeviceProfile> deviceListener) {
mainCtx.getDeviceProfileCache().addListener(getTenantId(), getSelfId(), profileListener, deviceListener);
} }
@Override @Override

7
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java

@ -23,6 +23,7 @@ import org.thingsboard.server.actors.TbActorRef;
import org.thingsboard.server.actors.TbEntityActorId; import org.thingsboard.server.actors.TbEntityActorId;
import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.service.DefaultActorService;
import org.thingsboard.server.actors.shared.ComponentMsgProcessor; import org.thingsboard.server.actors.shared.ComponentMsgProcessor;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleChainId;
@ -46,6 +47,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueCallback;
import org.thingsboard.server.queue.common.MultipleTbQueueTbMsgCallbackWrapper; import org.thingsboard.server.queue.common.MultipleTbQueueTbMsgCallbackWrapper;
import org.thingsboard.server.queue.common.TbQueueTbMsgCallbackWrapper; import org.thingsboard.server.queue.common.TbQueueTbMsgCallbackWrapper;
import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.queue.TbClusterService;
import java.util.ArrayList; import java.util.ArrayList;
@ -68,15 +70,16 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
private final Map<RuleNodeId, List<RuleNodeRelation>> nodeRoutes; private final Map<RuleNodeId, List<RuleNodeRelation>> nodeRoutes;
private final RuleChainService service; private final RuleChainService service;
private final TbClusterService clusterService; private final TbClusterService clusterService;
private final TbApiUsageClient apiUsageClient;
private String ruleChainName; private String ruleChainName;
private RuleNodeId firstId; private RuleNodeId firstId;
private RuleNodeCtx firstNode; private RuleNodeCtx firstNode;
private boolean started; private boolean started;
RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext, TbActorRef parent, TbActorRef self) {
, TbActorRef parent, TbActorRef self) {
super(systemContext, tenantId, ruleChain.getId()); super(systemContext, tenantId, ruleChain.getId());
this.apiUsageClient = systemContext.getApiUsageClient();
this.ruleChainName = ruleChain.getName(); this.ruleChainName = ruleChain.getName();
this.parent = parent; this.parent = parent;
this.self = self; this.self = self;

6
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java

@ -64,6 +64,12 @@ public abstract class RuleChainManagerActor extends ContextAwareActor {
} }
} }
protected void destroyRuleChains() {
for (RuleChain ruleChain : new PageDataIterable<>(link -> ruleChainService.findTenantRuleChains(tenantId, link), ContextAwareActor.ENTITY_PACK_LIMIT)) {
ctx.stop(new TbEntityActorId(ruleChain.getId()));
}
}
protected void visit(RuleChain entity, TbActorRef actorRef) { protected void visit(RuleChain entity, TbActorRef actorRef) {
if (entity != null && entity.isRoot()) { if (entity != null && entity.isRoot()) {
rootChain = entity; rootChain = entity;

51
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java

@ -21,13 +21,18 @@ import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorCtx;
import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.TbActorRef;
import org.thingsboard.server.actors.shared.ComponentMsgProcessor; import org.thingsboard.server.actors.shared.ComponentMsgProcessor;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; import org.thingsboard.server.common.data.plugin.ComponentLifecycleState;
import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.PartitionChangeMsg;
import org.thingsboard.server.common.msg.queue.RuleNodeException; import org.thingsboard.server.common.msg.queue.RuleNodeException;
import org.thingsboard.server.common.msg.queue.RuleNodeInfo; import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
/** /**
* @author Andrew Shvayka * @author Andrew Shvayka
@ -36,6 +41,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
private final String ruleChainName; private final String ruleChainName;
private final TbActorRef self; private final TbActorRef self;
private final TbApiUsageClient apiUsageClient;
private RuleNode ruleNode; private RuleNode ruleNode;
private TbNode tbNode; private TbNode tbNode;
private DefaultTbContext defaultCtx; private DefaultTbContext defaultCtx;
@ -44,6 +50,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
RuleNodeActorMessageProcessor(TenantId tenantId, String ruleChainName, RuleNodeId ruleNodeId, ActorSystemContext systemContext RuleNodeActorMessageProcessor(TenantId tenantId, String ruleChainName, RuleNodeId ruleNodeId, ActorSystemContext systemContext
, TbActorRef parent, TbActorRef self) { , TbActorRef parent, TbActorRef self) {
super(systemContext, tenantId, ruleNodeId); super(systemContext, tenantId, ruleNodeId);
this.apiUsageClient = systemContext.getApiUsageClient();
this.ruleChainName = ruleChainName; this.ruleChainName = ruleChainName;
this.self = self; this.self = self;
this.ruleNode = systemContext.getRuleChainService().findRuleNodeById(tenantId, entityId); this.ruleNode = systemContext.getRuleChainService().findRuleNodeById(tenantId, entityId);
@ -92,26 +99,42 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
public void onRuleToSelfMsg(RuleNodeToSelfMsg msg) throws Exception { public void onRuleToSelfMsg(RuleNodeToSelfMsg msg) throws Exception {
checkActive(msg.getMsg()); checkActive(msg.getMsg());
if (ruleNode.isDebugMode()) { TbMsg tbMsg = msg.getMsg();
systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), "Self"); int ruleNodeCount = tbMsg.getAndIncrementRuleNodeCounter();
} int maxRuleNodeExecutionsPerMessage = getTenantProfileConfiguration().getMaxRuleNodeExecsPerMessage();
try { if (maxRuleNodeExecutionsPerMessage == 0 || ruleNodeCount < maxRuleNodeExecutionsPerMessage) {
tbNode.onMsg(defaultCtx, msg.getMsg()); apiUsageClient.report(tenantId, ApiUsageRecordKey.RE_EXEC_COUNT);
} catch (Exception e) { if (ruleNode.isDebugMode()) {
defaultCtx.tellFailure(msg.getMsg(), e); systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), "Self");
}
try {
tbNode.onMsg(defaultCtx, msg.getMsg());
} catch (Exception e) {
defaultCtx.tellFailure(msg.getMsg(), e);
}
} else {
tbMsg.getCallback().onFailure(new RuleNodeException("Message is processed by more then " + maxRuleNodeExecutionsPerMessage + " rule nodes!", ruleChainName, ruleNode));
} }
} }
void onRuleChainToRuleNodeMsg(RuleChainToRuleNodeMsg msg) throws Exception { void onRuleChainToRuleNodeMsg(RuleChainToRuleNodeMsg msg) throws Exception {
msg.getMsg().getCallback().onProcessingStart(info); msg.getMsg().getCallback().onProcessingStart(info);
checkActive(msg.getMsg()); checkActive(msg.getMsg());
if (ruleNode.isDebugMode()) { TbMsg tbMsg = msg.getMsg();
systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), msg.getFromRelationType()); int ruleNodeCount = tbMsg.getAndIncrementRuleNodeCounter();
} int maxRuleNodeExecutionsPerMessage = getTenantProfileConfiguration().getMaxRuleNodeExecsPerMessage();
try { if (maxRuleNodeExecutionsPerMessage == 0 || ruleNodeCount < maxRuleNodeExecutionsPerMessage) {
tbNode.onMsg(msg.getCtx(), msg.getMsg()); apiUsageClient.report(tenantId, ApiUsageRecordKey.RE_EXEC_COUNT);
} catch (Exception e) { if (ruleNode.isDebugMode()) {
msg.getCtx().tellFailure(msg.getMsg(), e); systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), msg.getFromRelationType());
}
try {
tbNode.onMsg(msg.getCtx(), msg.getMsg());
} catch (Exception e) {
msg.getCtx().tellFailure(msg.getMsg(), e);
}
} else {
tbMsg.getCallback().onFailure(new RuleNodeException("Message is processed by more then " + maxRuleNodeExecutionsPerMessage + " rule nodes!", ruleChainName, ruleNode));
} }
} }

2
application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java

@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener; import org.springframework.context.event.EventListener;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.ActorSystemContext;
@ -113,6 +114,7 @@ public class DefaultActorService implements ActorService {
} }
@EventListener(ApplicationReadyEvent.class) @EventListener(ApplicationReadyEvent.class)
@Order(value = 2)
public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
log.info("Received application ready event. Sending application init message to actor system"); log.info("Received application ready event. Sending application init message to actor system");
appActor.tellWithHighPriority(new AppInitMsg()); appActor.tellWithHighPriority(new AppInitMsg());

6
application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java

@ -19,9 +19,11 @@ import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorCtx;
import org.thingsboard.server.actors.stats.StatsPersistTick; import org.thingsboard.server.actors.stats.StatsPersistTick;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; import org.thingsboard.server.common.data.plugin.ComponentLifecycleState;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.PartitionChangeMsg;
import org.thingsboard.server.common.msg.queue.RuleNodeException; import org.thingsboard.server.common.msg.queue.RuleNodeException;
@ -39,6 +41,10 @@ public abstract class ComponentMsgProcessor<T extends EntityId> extends Abstract
this.entityId = id; this.entityId = id;
} }
protected TenantProfileConfiguration getTenantProfileConfiguration() {
return systemContext.getTenantProfileCache().get(tenantId).getProfileData().getConfiguration();
}
public abstract String getComponentName(); public abstract String getComponentName();
public abstract void start(TbActorCtx context) throws Exception; public abstract void start(TbActorCtx context) throws Exception;

63
application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java

@ -29,6 +29,7 @@ import org.thingsboard.server.actors.device.DeviceActorCreator;
import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor;
import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.ContextBasedCreator;
import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.service.DefaultActorService;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.TenantProfile;
@ -57,6 +58,7 @@ public class TenantActor extends RuleChainManagerActor {
private boolean isRuleEngineForCurrentTenant; private boolean isRuleEngineForCurrentTenant;
private boolean isCore; private boolean isCore;
private ApiUsageState apiUsageState;
private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { private TenantActor(ActorSystemContext systemContext, TenantId tenantId) {
super(systemContext, tenantId); super(systemContext, tenantId);
@ -74,19 +76,24 @@ public class TenantActor extends RuleChainManagerActor {
cantFindTenant = true; cantFindTenant = true;
log.info("[{}] Started tenant actor for missing tenant.", tenantId); log.info("[{}] Started tenant actor for missing tenant.", tenantId);
} else { } else {
apiUsageState = new ApiUsageState(systemContext.getApiUsageStateService().getApiUsageState(tenant.getId()));
// This Service may be started for specific tenant only. // This Service may be started for specific tenant only.
Optional<TenantId> isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); Optional<TenantId> isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant();
TenantProfile tenantProfile = systemContext.getTenantProfileCache().get(tenant.getTenantProfileId()); TenantProfile tenantProfile = systemContext.getTenantProfileCache().get(tenant.getTenantProfileId());
isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE);
isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE);
isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE);
if (isRuleEngineForCurrentTenant) { if (isRuleEngineForCurrentTenant) {
try { try {
if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenantProfile.isIsolatedTbRuleEngine())) { if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenantProfile.isIsolatedTbRuleEngine())) {
log.info("[{}] Going to init rule chains", tenantId); if (apiUsageState.isReExecEnabled()) {
initRuleChains(); log.info("[{}] Going to init rule chains", tenantId);
initRuleChains();
} else {
log.info("[{}] Skip init of the rule chains due to API limits", tenantId);
}
} else { } else {
isRuleEngineForCurrentTenant = false; isRuleEngineForCurrentTenant = false;
} }
@ -98,8 +105,6 @@ public class TenantActor extends RuleChainManagerActor {
} }
} catch (Exception e) { } catch (Exception e) {
log.warn("[{}] Unknown failure", tenantId, e); log.warn("[{}] Unknown failure", tenantId, e);
// TODO: throw this in 3.1?
// throw new TbActorException("Failed to init actor", e);
} }
} }
@ -115,7 +120,7 @@ public class TenantActor extends RuleChainManagerActor {
if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) { if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) {
QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg; QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg;
queueMsg.getTbMsg().getCallback().onSuccess(); queueMsg.getTbMsg().getCallback().onSuccess();
} else if (msg.getMsgType().equals(MsgType.TRANSPORT_TO_DEVICE_ACTOR_MSG)){ } else if (msg.getMsgType().equals(MsgType.TRANSPORT_TO_DEVICE_ACTOR_MSG)) {
TransportToDeviceActorMsgWrapper transportMsg = (TransportToDeviceActorMsgWrapper) msg; TransportToDeviceActorMsgWrapper transportMsg = (TransportToDeviceActorMsgWrapper) msg;
transportMsg.getCallback().onSuccess(); transportMsg.getCallback().onSuccess();
} }
@ -173,26 +178,33 @@ public class TenantActor extends RuleChainManagerActor {
return; return;
} }
TbMsg tbMsg = msg.getTbMsg(); TbMsg tbMsg = msg.getTbMsg();
if (tbMsg.getRuleChainId() == null) { if (apiUsageState.isReExecEnabled()) {
if (getRootChainActor() != null) { if (tbMsg.getRuleChainId() == null) {
getRootChainActor().tell(msg); if (getRootChainActor() != null) {
getRootChainActor().tell(msg);
} else {
tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!"));
log.info("[{}] No Root Chain: {}", tenantId, msg);
}
} else { } else {
tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!")); try {
log.info("[{}] No Root Chain: {}", tenantId, msg); ctx.tell(new TbEntityActorId(tbMsg.getRuleChainId()), msg);
} catch (TbActorNotRegisteredException ex) {
log.trace("Received message for non-existing rule chain: [{}]", tbMsg.getRuleChainId());
//TODO: 3.1 Log it to dead letters queue;
tbMsg.getCallback().onSuccess();
}
} }
} else { } else {
try { log.trace("[{}] Ack message because Rule Engine is disabled", tenantId);
ctx.tell(new TbEntityActorId(tbMsg.getRuleChainId()), msg); tbMsg.getCallback().onSuccess();
} catch (TbActorNotRegisteredException ex) {
log.trace("Received message for non-existing rule chain: [{}]", tbMsg.getRuleChainId());
//TODO: 3.1 Log it to dead letters queue;
tbMsg.getCallback().onSuccess();
}
} }
} }
private void onRuleChainMsg(RuleChainAwareMsg msg) { private void onRuleChainMsg(RuleChainAwareMsg msg) {
getOrCreateActor(msg.getRuleChainId()).tell(msg); if (apiUsageState.isReExecEnabled()) {
getOrCreateActor(msg.getRuleChainId()).tell(msg);
}
} }
private void onToDeviceActorMsg(DeviceAwareMsg msg, boolean priority) { private void onToDeviceActorMsg(DeviceAwareMsg msg, boolean priority) {
@ -208,6 +220,17 @@ public class TenantActor extends RuleChainManagerActor {
} }
private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) {
if (msg.getEntityId().getEntityType().equals(EntityType.API_USAGE_STATE)) {
ApiUsageState old = apiUsageState;
apiUsageState = new ApiUsageState(systemContext.getApiUsageStateService().getApiUsageState(tenantId));
if (old.isReExecEnabled() && !apiUsageState.isReExecEnabled()) {
log.info("[{}] Received API state update. Going to DISABLE Rule Engine execution.", tenantId);
destroyRuleChains();
} else if (!old.isReExecEnabled() && apiUsageState.isReExecEnabled()) {
log.info("[{}] Received API state update. Going to ENABLE Rule Engine execution.", tenantId);
initRuleChains();
}
}
if (isRuleEngineForCurrentTenant) { if (isRuleEngineForCurrentTenant) {
TbActorRef target = getEntityActorRef(msg.getEntityId()); TbActorRef target = getEntityActorRef(msg.getEntityId());
if (target != null) { if (target != null) {

2
application/src/main/java/org/thingsboard/server/controller/AlarmController.java

@ -126,6 +126,7 @@ public class AlarmController extends BaseController {
long ackTs = System.currentTimeMillis(); long ackTs = System.currentTimeMillis();
alarmService.ackAlarm(getCurrentUser().getTenantId(), alarmId, ackTs).get(); alarmService.ackAlarm(getCurrentUser().getTenantId(), alarmId, ackTs).get();
alarm.setAckTs(ackTs); alarm.setAckTs(ackTs);
alarm.setStatus(alarm.getStatus().isCleared() ? AlarmStatus.CLEARED_ACK : AlarmStatus.ACTIVE_ACK);
logEntityAction(alarm.getOriginator(), alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_ACK, null); logEntityAction(alarm.getOriginator(), alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_ACK, null);
} catch (Exception e) { } catch (Exception e) {
throw handleException(e); throw handleException(e);
@ -143,6 +144,7 @@ public class AlarmController extends BaseController {
long clearTs = System.currentTimeMillis(); long clearTs = System.currentTimeMillis();
alarmService.clearAlarm(getCurrentUser().getTenantId(), alarmId, null, clearTs).get(); alarmService.clearAlarm(getCurrentUser().getTenantId(), alarmId, null, clearTs).get();
alarm.setClearTs(clearTs); alarm.setClearTs(clearTs);
alarm.setStatus(alarm.getStatus().isAck() ? AlarmStatus.CLEARED_ACK : AlarmStatus.CLEARED_UNACK);
logEntityAction(alarm.getOriginator(), alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_CLEAR, null); logEntityAction(alarm.getOriginator(), alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_CLEAR, null);
} catch (Exception e) { } catch (Exception e) {
throw handleException(e); throw handleException(e);

7
application/src/main/java/org/thingsboard/server/controller/AuthController.java

@ -166,7 +166,8 @@ public class AuthController extends BaseController {
try { try {
String email = resetPasswordByEmailRequest.get("email").asText(); String email = resetPasswordByEmailRequest.get("email").asText();
UserCredentials userCredentials = userService.requestPasswordReset(TenantId.SYS_TENANT_ID, email); UserCredentials userCredentials = userService.requestPasswordReset(TenantId.SYS_TENANT_ID, email);
String baseUrl = MiscUtils.constructBaseUrl(request); User user = userService.findUserById(TenantId.SYS_TENANT_ID, userCredentials.getUserId());
String baseUrl = systemSecurityService.getBaseUrl(user.getTenantId(), user.getCustomerId(), request);
String resetUrl = String.format("%s/api/noauth/resetPassword?resetToken=%s", baseUrl, String resetUrl = String.format("%s/api/noauth/resetPassword?resetToken=%s", baseUrl,
userCredentials.getResetToken()); userCredentials.getResetToken());
@ -214,7 +215,7 @@ public class AuthController extends BaseController {
User user = userService.findUserById(TenantId.SYS_TENANT_ID, credentials.getUserId()); User user = userService.findUserById(TenantId.SYS_TENANT_ID, credentials.getUserId());
UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail()); UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail());
SecurityUser securityUser = new SecurityUser(user, credentials.isEnabled(), principal); SecurityUser securityUser = new SecurityUser(user, credentials.isEnabled(), principal);
String baseUrl = MiscUtils.constructBaseUrl(request); String baseUrl = systemSecurityService.getBaseUrl(user.getTenantId(), user.getCustomerId(), request);
String loginUrl = String.format("%s/login", baseUrl); String loginUrl = String.format("%s/login", baseUrl);
String email = user.getEmail(); String email = user.getEmail();
@ -261,7 +262,7 @@ public class AuthController extends BaseController {
User user = userService.findUserById(TenantId.SYS_TENANT_ID, userCredentials.getUserId()); User user = userService.findUserById(TenantId.SYS_TENANT_ID, userCredentials.getUserId());
UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail()); UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail());
SecurityUser securityUser = new SecurityUser(user, userCredentials.isEnabled(), principal); SecurityUser securityUser = new SecurityUser(user, userCredentials.isEnabled(), principal);
String baseUrl = MiscUtils.constructBaseUrl(request); String baseUrl = systemSecurityService.getBaseUrl(user.getTenantId(), user.getCustomerId(), request);
String loginUrl = String.format("%s/login", baseUrl); String loginUrl = String.format("%s/login", baseUrl);
String email = user.getEmail(); String email = user.getEmail();
mailService.sendPasswordWasResetEmail(loginUrl, email); mailService.sendPasswordWasResetEmail(loginUrl, email);

3
application/src/main/java/org/thingsboard/server/controller/BaseController.java

@ -35,7 +35,6 @@ import org.thingsboard.server.common.data.asset.AssetInfo;
import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.*;
import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AlarmId;
import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.CustomerId;
@ -94,7 +93,7 @@ import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.service.profile.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.queue.TbClusterService;
import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.AccessControlService;

3
application/src/main/java/org/thingsboard/server/controller/DeviceController.java

@ -51,6 +51,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgDataType;
@ -119,6 +120,8 @@ public class DeviceController extends BaseController {
tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(),
savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null); savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null);
tbClusterService.onEntityStateChange(savedDevice.getTenantId(), savedDevice.getId(),
device.getId() == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
logEntityAction(savedDevice.getId(), savedDevice, logEntityAction(savedDevice.getId(), savedDevice,
savedDevice.getCustomerId(), savedDevice.getCustomerId(),

4
application/src/main/java/org/thingsboard/server/controller/RuleChainController.java

@ -160,7 +160,7 @@ public class RuleChainController extends BaseController {
public RuleChain saveRuleChain(@RequestBody DefaultRuleChainCreateRequest request) throws ThingsboardException { public RuleChain saveRuleChain(@RequestBody DefaultRuleChainCreateRequest request) throws ThingsboardException {
try { try {
checkNotNull(request); checkNotNull(request);
checkNotNull(request.getName()); checkParameter(request.getName(), "name");
RuleChain savedRuleChain = installScripts.createDefaultRuleChain(getCurrentUser().getTenantId(), request.getName()); RuleChain savedRuleChain = installScripts.createDefaultRuleChain(getCurrentUser().getTenantId(), request.getName());
@ -347,7 +347,7 @@ public class RuleChainController extends BaseController {
String errorText = ""; String errorText = "";
ScriptEngine engine = null; ScriptEngine engine = null;
try { try {
engine = new RuleNodeJsScriptEngine(jsInvokeService, getCurrentUser().getId(), script, argNames); engine = new RuleNodeJsScriptEngine(getTenantId(), jsInvokeService, getCurrentUser().getId(), script, argNames);
TbMsg inMsg = TbMsg.newMsg(msgType, null, new TbMsgMetaData(metadata), TbMsgDataType.JSON, data); TbMsg inMsg = TbMsg.newMsg(msgType, null, new TbMsgMetaData(metadata), TbMsgDataType.JSON, data);
switch (scriptType) { switch (scriptType) {
case "update": case "update":

5
application/src/main/java/org/thingsboard/server/controller/TelemetryController.java

@ -392,6 +392,11 @@ public class TelemetryController extends BaseController {
if (attributes.isEmpty()) { if (attributes.isEmpty()) {
return getImmediateDeferredResult("No attributes data found in request body!", HttpStatus.BAD_REQUEST); return getImmediateDeferredResult("No attributes data found in request body!", HttpStatus.BAD_REQUEST);
} }
for (AttributeKvEntry attributeKvEntry: attributes) {
if (attributeKvEntry.getKey().isEmpty() || attributeKvEntry.getKey().trim().length() == 0) {
return getImmediateDeferredResult("Key cannot be empty or contains only spaces", HttpStatus.BAD_REQUEST);
}
}
SecurityUser user = getCurrentUser(); SecurityUser user = getCurrentUser();
return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdSrc, (result, tenantId, entityId) -> { return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdSrc, (result, tenantId, entityId) -> {
tsSubService.saveAndNotify(tenantId, entityId, scope, attributes, new FutureCallback<Void>() { tsSubService.saveAndNotify(tenantId, entityId, scope, attributes, new FutureCallback<Void>() {

2
application/src/main/java/org/thingsboard/server/controller/TenantController.java

@ -93,6 +93,8 @@ public class TenantController extends BaseController {
} }
tenantProfileCache.evict(tenant.getId()); tenantProfileCache.evict(tenant.getId());
tbClusterService.onTenantChange(tenant, null); tbClusterService.onTenantChange(tenant, null);
tbClusterService.onEntityStateChange(tenant.getId(), tenant.getId(),
newTenant ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
return tenant; return tenant;
} catch (Exception e) { } catch (Exception e) {
throw handleException(e); throw handleException(e);

10
application/src/main/java/org/thingsboard/server/controller/UserController.java

@ -52,6 +52,7 @@ import org.thingsboard.server.service.security.model.token.JwtToken;
import org.thingsboard.server.service.security.model.token.JwtTokenFactory; import org.thingsboard.server.service.security.model.token.JwtTokenFactory;
import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.permission.Resource;
import org.thingsboard.server.service.security.system.SystemSecurityService;
import org.thingsboard.server.utils.MiscUtils; import org.thingsboard.server.utils.MiscUtils;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
@ -78,6 +79,9 @@ public class UserController extends BaseController {
@Autowired @Autowired
private RefreshTokenRepository refreshTokenRepository; private RefreshTokenRepository refreshTokenRepository;
@Autowired
private SystemSecurityService systemSecurityService;
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
@RequestMapping(value = "/user/{userId}", method = RequestMethod.GET) @RequestMapping(value = "/user/{userId}", method = RequestMethod.GET)
@ -146,7 +150,7 @@ public class UserController extends BaseController {
if (sendEmail) { if (sendEmail) {
SecurityUser authUser = getCurrentUser(); SecurityUser authUser = getCurrentUser();
UserCredentials userCredentials = userService.findUserCredentialsByUserId(authUser.getTenantId(), savedUser.getId()); UserCredentials userCredentials = userService.findUserCredentialsByUserId(authUser.getTenantId(), savedUser.getId());
String baseUrl = MiscUtils.constructBaseUrl(request); String baseUrl = systemSecurityService.getBaseUrl(getTenantId(), getCurrentUser().getCustomerId(), request);
String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl, String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl,
userCredentials.getActivateToken()); userCredentials.getActivateToken());
String email = savedUser.getEmail(); String email = savedUser.getEmail();
@ -186,7 +190,7 @@ public class UserController extends BaseController {
UserCredentials userCredentials = userService.findUserCredentialsByUserId(getCurrentUser().getTenantId(), user.getId()); UserCredentials userCredentials = userService.findUserCredentialsByUserId(getCurrentUser().getTenantId(), user.getId());
if (!userCredentials.isEnabled()) { if (!userCredentials.isEnabled()) {
String baseUrl = MiscUtils.constructBaseUrl(request); String baseUrl = systemSecurityService.getBaseUrl(getTenantId(), getCurrentUser().getCustomerId(), request);
String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl, String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl,
userCredentials.getActivateToken()); userCredentials.getActivateToken());
mailService.sendActivationEmail(activateUrl, email); mailService.sendActivationEmail(activateUrl, email);
@ -211,7 +215,7 @@ public class UserController extends BaseController {
SecurityUser authUser = getCurrentUser(); SecurityUser authUser = getCurrentUser();
UserCredentials userCredentials = userService.findUserCredentialsByUserId(authUser.getTenantId(), user.getId()); UserCredentials userCredentials = userService.findUserCredentialsByUserId(authUser.getTenantId(), user.getId());
if (!userCredentials.isEnabled()) { if (!userCredentials.isEnabled()) {
String baseUrl = MiscUtils.constructBaseUrl(request); String baseUrl = systemSecurityService.getBaseUrl(getTenantId(), getCurrentUser().getCustomerId(), request);
String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl, String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl,
userCredentials.getActivateToken()); userCredentials.getActivateToken());
return activateUrl; return activateUrl;

370
application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java

@ -0,0 +1,370 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.apiusage;
import com.google.common.util.concurrent.FutureCallback;
import lombok.extern.slf4j.Slf4j;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.event.EventListener;
import org.springframework.core.annotation.Order;
import org.springframework.data.util.Pair;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.ApiFeature;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.ApiUsageStateValue;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.ApiUsageStateId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.TenantProfileId;
import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.kv.StringDataEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.data.page.PageDataIterable;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.tools.SchedulerUtils;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.scheduler.SchedulerComponent;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.service.queue.TbClusterService;
import org.thingsboard.server.service.telemetry.InternalTelemetryService;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@Slf4j
@Service
public class DefaultTbApiUsageStateService implements TbApiUsageStateService {
public static final String HOURLY = "Hourly";
public static final FutureCallback<Integer> VOID_CALLBACK = new FutureCallback<Integer>() {
@Override
public void onSuccess(@Nullable Integer result) {
}
@Override
public void onFailure(Throwable t) {
}
};
private final TbClusterService clusterService;
private final PartitionService partitionService;
private final TenantService tenantService;
private final TimeseriesService tsService;
private final ApiUsageStateService apiUsageStateService;
private final SchedulerComponent scheduler;
private final TbTenantProfileCache tenantProfileCache;
@Lazy
@Autowired
private InternalTelemetryService tsWsService;
// Tenants that should be processed on this server
private final Map<TenantId, TenantApiUsageState> myTenantStates = new ConcurrentHashMap<>();
// Tenants that should be processed on other servers
private final Map<TenantId, ApiUsageState> otherTenantStates = new ConcurrentHashMap<>();
@Value("${usage.stats.report.enabled:true}")
private boolean enabled;
@Value("${usage.stats.check.cycle:60000}")
private long nextCycleCheckInterval;
private final Lock updateLock = new ReentrantLock();
public DefaultTbApiUsageStateService(TbClusterService clusterService,
PartitionService partitionService,
TenantService tenantService,
TimeseriesService tsService,
ApiUsageStateService apiUsageStateService,
SchedulerComponent scheduler,
TbTenantProfileCache tenantProfileCache) {
this.clusterService = clusterService;
this.partitionService = partitionService;
this.tenantService = tenantService;
this.tsService = tsService;
this.apiUsageStateService = apiUsageStateService;
this.scheduler = scheduler;
this.tenantProfileCache = tenantProfileCache;
}
@PostConstruct
public void init() {
if (enabled) {
log.info("Starting api usage service.");
scheduler.scheduleAtFixedRate(this::checkStartOfNextCycle, nextCycleCheckInterval, nextCycleCheckInterval, TimeUnit.MILLISECONDS);
log.info("Started api usage service.");
}
}
@Override
public void process(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback) {
ToUsageStatsServiceMsg statsMsg = msg.getValue();
TenantId tenantId = new TenantId(new UUID(statsMsg.getTenantIdMSB(), statsMsg.getTenantIdLSB()));
TenantApiUsageState tenantState;
List<TsKvEntry> updatedEntries;
Map<ApiFeature, ApiUsageStateValue> result;
updateLock.lock();
try {
tenantState = getOrFetchState(tenantId);
long ts = tenantState.getCurrentCycleTs();
long hourTs = tenantState.getCurrentHourTs();
long newHourTs = SchedulerUtils.getStartOfCurrentHour();
if (newHourTs != hourTs) {
tenantState.setHour(newHourTs);
}
updatedEntries = new ArrayList<>(ApiUsageRecordKey.values().length);
Set<ApiFeature> apiFeatures = new HashSet<>();
for (UsageStatsKVProto kvProto : statsMsg.getValuesList()) {
ApiUsageRecordKey recordKey = ApiUsageRecordKey.valueOf(kvProto.getKey());
long newValue = tenantState.add(recordKey, kvProto.getValue());
updatedEntries.add(new BasicTsKvEntry(ts, new LongDataEntry(recordKey.getApiCountKey(), newValue)));
long newHourlyValue = tenantState.addToHourly(recordKey, kvProto.getValue());
updatedEntries.add(new BasicTsKvEntry(hourTs, new LongDataEntry(recordKey.getApiCountKey() + HOURLY, newHourlyValue)));
apiFeatures.add(recordKey.getApiFeature());
}
result = tenantState.checkStateUpdatedDueToThreshold(apiFeatures);
} finally {
updateLock.unlock();
}
tsWsService.saveAndNotifyInternal(tenantId, tenantState.getApiUsageState().getId(), updatedEntries, VOID_CALLBACK);
if (!result.isEmpty()) {
persistAndNotify(tenantState, result);
}
callback.onSuccess();
}
@Override
public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
if (partitionChangeEvent.getServiceType().equals(ServiceType.TB_CORE)) {
myTenantStates.entrySet().removeIf(entry -> !partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition());
otherTenantStates.entrySet().removeIf(entry -> partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition());
initStatesFromDataBase();
}
}
@Override
public ApiUsageState getApiUsageState(TenantId tenantId) {
TenantApiUsageState tenantState = myTenantStates.get(tenantId);
if (tenantState != null) {
return tenantState.getApiUsageState();
} else {
ApiUsageState state = otherTenantStates.get(tenantId);
if (state != null) {
return state;
} else {
if (partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) {
return getOrFetchState(tenantId).getApiUsageState();
} else {
updateLock.lock();
try {
state = otherTenantStates.get(tenantId);
if (state == null) {
state = apiUsageStateService.findTenantApiUsageState(tenantId);
if (state != null) {
otherTenantStates.put(tenantId, state);
}
}
} finally {
updateLock.unlock();
}
return state;
}
}
}
}
@Override
public void onApiUsageStateUpdate(TenantId tenantId) {
otherTenantStates.remove(tenantId);
}
@Override
public void onTenantProfileUpdate(TenantProfileId tenantProfileId) {
log.info("[{}] On Tenant Profile Update", tenantProfileId);
TenantProfile tenantProfile = tenantProfileCache.get(tenantProfileId);
updateLock.lock();
try {
myTenantStates.values().forEach(state -> {
if (tenantProfile.getId().equals(state.getTenantProfileId())) {
updateTenantState(state, tenantProfile);
}
});
} finally {
updateLock.unlock();
}
}
@Override
public void onTenantUpdate(TenantId tenantId) {
log.info("[{}] On Tenant Update.", tenantId);
TenantProfile tenantProfile = tenantProfileCache.get(tenantId);
updateLock.lock();
try {
TenantApiUsageState state = myTenantStates.get(tenantId);
if (state != null && !state.getTenantProfileId().equals(tenantProfile.getId())) {
updateTenantState(state, tenantProfile);
}
} finally {
updateLock.unlock();
}
}
private void updateTenantState(TenantApiUsageState state, TenantProfile profile) {
TenantProfileData oldProfileData = state.getTenantProfileData();
state.setTenantProfileId(profile.getId());
state.setTenantProfileData(profile.getProfileData());
Map<ApiFeature, ApiUsageStateValue> result = state.checkStateUpdatedDueToThresholds();
if (!result.isEmpty()) {
persistAndNotify(state, result);
}
updateProfileThresholds(state.getTenantId(), state.getApiUsageState().getId(),
oldProfileData.getConfiguration(), profile.getProfileData().getConfiguration());
}
private void updateProfileThresholds(TenantId tenantId, ApiUsageStateId id,
TenantProfileConfiguration oldData, TenantProfileConfiguration newData) {
long ts = System.currentTimeMillis();
List<TsKvEntry> profileThresholds = new ArrayList<>();
for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
long newProfileThreshold = newData.getProfileThreshold(key);
if (oldData == null || oldData.getProfileThreshold(key) != newProfileThreshold) {
log.info("[{}] Updating profile threshold [{}]:[{}]", tenantId, key, newProfileThreshold);
profileThresholds.add(new BasicTsKvEntry(ts, new LongDataEntry(key.getApiLimitKey(), newProfileThreshold)));
}
}
if (!profileThresholds.isEmpty()) {
tsWsService.saveAndNotifyInternal(tenantId, id, profileThresholds, VOID_CALLBACK);
}
}
private void persistAndNotify(TenantApiUsageState state, Map<ApiFeature, ApiUsageStateValue> result) {
log.info("[{}] Detected update of the API state: {}", state.getTenantId(), result);
apiUsageStateService.update(state.getApiUsageState());
clusterService.onApiStateChange(state.getApiUsageState(), null);
long ts = System.currentTimeMillis();
List<TsKvEntry> stateTelemetry = new ArrayList<>();
result.forEach(((apiFeature, aState) -> stateTelemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(apiFeature.getApiStateKey(), aState.name())))));
tsWsService.saveAndNotifyInternal(state.getTenantId(), state.getApiUsageState().getId(), stateTelemetry, VOID_CALLBACK);
//TODO: notify tenant admin via email!
}
private void checkStartOfNextCycle() {
updateLock.lock();
try {
long now = System.currentTimeMillis();
myTenantStates.values().forEach(state -> {
if ((state.getNextCycleTs() > now) && (state.getNextCycleTs() - now < TimeUnit.HOURS.toMillis(1))) {
state.setCycles(state.getNextCycleTs(), SchedulerUtils.getStartOfNextNextMonth());
}
});
} finally {
updateLock.unlock();
}
}
private TenantApiUsageState getOrFetchState(TenantId tenantId) {
TenantApiUsageState tenantState = myTenantStates.get(tenantId);
if (tenantState == null) {
ApiUsageState dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId);
if (dbStateEntity == null) {
try {
dbStateEntity = apiUsageStateService.createDefaultApiUsageState(tenantId);
} catch (Exception e) {
dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId);
}
}
TenantProfile tenantProfile = tenantProfileCache.get(tenantId);
tenantState = new TenantApiUsageState(tenantProfile, dbStateEntity);
try {
List<TsKvEntry> dbValues = tsService.findAllLatest(tenantId, dbStateEntity.getId()).get();
for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
boolean cycleEntryFound = false;
boolean hourlyEntryFound = false;
for (TsKvEntry tsKvEntry : dbValues) {
if (tsKvEntry.getKey().equals(key.getApiCountKey())) {
cycleEntryFound = true;
tenantState.put(key, tsKvEntry.getTs() == tenantState.getCurrentCycleTs() ? tsKvEntry.getLongValue().get() : 0L);
} else if (tsKvEntry.getKey().equals(key.getApiCountKey() + HOURLY)) {
hourlyEntryFound = true;
tenantState.putHourly(key, tsKvEntry.getTs() == tenantState.getCurrentHourTs() ? tsKvEntry.getLongValue().get() : 0L);
}
if (cycleEntryFound && hourlyEntryFound) {
break;
}
}
}
log.debug("[{}] Initialized state: {}", tenantId, dbStateEntity);
myTenantStates.put(tenantId, tenantState);
} catch (InterruptedException | ExecutionException e) {
log.warn("[{}] Failed to fetch api usage state from db.", tenantId, e);
}
}
return tenantState;
}
private void initStatesFromDataBase() {
try {
log.info("Initializing tenant states.");
PageDataIterable<Tenant> tenantIterator = new PageDataIterable<>(tenantService::findTenants, 1024);
for (Tenant tenant : tenantIterator) {
if (!myTenantStates.containsKey(tenant.getId()) && partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), tenant.getId()).isMyPartition()) {
log.debug("[{}] Initializing tenant state.", tenant.getId());
updateLock.lock();
try {
updateTenantState(getOrFetchState(tenant.getId()), tenantProfileCache.get(tenant.getTenantProfileId()));
log.debug("[{}] Initialized tenant state.", tenant.getId());
} catch (Exception e) {
log.warn("[{}] Failed to initialize tenant API state", tenant.getId(), e);
} finally {
updateLock.unlock();
}
}
}
log.info("Initialized tenant states.");
} catch (Exception e) {
log.warn("Unknown failure", e);
}
}
}

38
application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java

@ -0,0 +1,38 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.apiusage;
import org.springframework.context.ApplicationListener;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.TenantProfileId;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
public interface TbApiUsageStateService extends ApplicationListener<PartitionChangeEvent> {
void process(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback);
ApiUsageState getApiUsageState(TenantId tenantId);
void onTenantProfileUpdate(TenantProfileId tenantProfileId);
void onTenantUpdate(TenantId tenantId);
void onApiUsageStateUpdate(TenantId tenantId);
}

186
application/src/main/java/org/thingsboard/server/service/apiusage/TenantApiUsageState.java

@ -0,0 +1,186 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.apiusage;
import lombok.Getter;
import lombok.Setter;
import org.springframework.data.util.Pair;
import org.thingsboard.server.common.data.ApiFeature;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.ApiUsageStateValue;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.TenantProfileId;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import org.thingsboard.server.common.msg.tools.SchedulerUtils;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
public class TenantApiUsageState {
private final Map<ApiUsageRecordKey, Long> currentCycleValues = new ConcurrentHashMap<>();
private final Map<ApiUsageRecordKey, Long> currentHourValues = new ConcurrentHashMap<>();
@Getter
@Setter
private TenantProfileId tenantProfileId;
@Getter
@Setter
private TenantProfileData tenantProfileData;
@Getter
private final ApiUsageState apiUsageState;
@Getter
private volatile long currentCycleTs;
@Getter
private volatile long nextCycleTs;
@Getter
private volatile long currentHourTs;
public TenantApiUsageState(TenantProfile tenantProfile, ApiUsageState apiUsageState) {
this.tenantProfileId = tenantProfile.getId();
this.tenantProfileData = tenantProfile.getProfileData();
this.apiUsageState = apiUsageState;
this.currentCycleTs = SchedulerUtils.getStartOfCurrentMonth();
this.nextCycleTs = SchedulerUtils.getStartOfNextMonth();
this.currentHourTs = SchedulerUtils.getStartOfCurrentHour();
}
public void put(ApiUsageRecordKey key, Long value) {
currentCycleValues.put(key, value);
}
public void putHourly(ApiUsageRecordKey key, Long value) {
currentHourValues.put(key, value);
}
public long add(ApiUsageRecordKey key, long value) {
long result = currentCycleValues.getOrDefault(key, 0L) + value;
currentCycleValues.put(key, result);
return result;
}
public long get(ApiUsageRecordKey key) {
return currentCycleValues.getOrDefault(key, 0L);
}
public long addToHourly(ApiUsageRecordKey key, long value) {
long result = currentHourValues.getOrDefault(key, 0L) + value;
currentHourValues.put(key, result);
return result;
}
public void setHour(long currentHourTs) {
this.currentHourTs = currentHourTs;
for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
currentHourValues.put(key, 0L);
}
}
public void setCycles(long currentCycleTs, long nextCycleTs) {
this.currentCycleTs = currentCycleTs;
this.nextCycleTs = nextCycleTs;
for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
currentCycleValues.put(key, 0L);
}
}
public long getProfileThreshold(ApiUsageRecordKey key) {
return tenantProfileData.getConfiguration().getProfileThreshold(key);
}
public long getProfileWarnThreshold(ApiUsageRecordKey key) {
return tenantProfileData.getConfiguration().getWarnThreshold(key);
}
public TenantId getTenantId() {
return apiUsageState.getTenantId();
}
public ApiUsageStateValue getFeatureValue(ApiFeature feature) {
switch (feature) {
case TRANSPORT:
return apiUsageState.getTransportState();
case RE:
return apiUsageState.getReExecState();
case DB:
return apiUsageState.getDbStorageState();
case JS:
return apiUsageState.getJsExecState();
default:
return ApiUsageStateValue.ENABLED;
}
}
public boolean setFeatureValue(ApiFeature feature, ApiUsageStateValue value) {
ApiUsageStateValue currentValue = getFeatureValue(feature);
switch (feature) {
case TRANSPORT:
apiUsageState.setTransportState(value);
break;
case RE:
apiUsageState.setReExecState(value);
break;
case DB:
apiUsageState.setDbStorageState(value);
break;
case JS:
apiUsageState.setJsExecState(value);
break;
}
return !currentValue.equals(value);
}
public Map<ApiFeature, ApiUsageStateValue> checkStateUpdatedDueToThresholds() {
return checkStateUpdatedDueToThreshold(new HashSet<>(Arrays.asList(ApiFeature.values())));
}
public Map<ApiFeature, ApiUsageStateValue> checkStateUpdatedDueToThreshold(Set<ApiFeature> features) {
Map<ApiFeature, ApiUsageStateValue> result = new HashMap<>();
for (ApiFeature feature : features) {
Pair<ApiFeature, ApiUsageStateValue> tmp = checkStateUpdatedDueToThreshold(feature);
if (tmp != null) {
result.put(tmp.getFirst(), tmp.getSecond());
}
}
return result;
}
public Pair<ApiFeature, ApiUsageStateValue> checkStateUpdatedDueToThreshold(ApiFeature feature) {
ApiUsageStateValue featureValue = ApiUsageStateValue.ENABLED;
for (ApiUsageRecordKey recordKey : ApiUsageRecordKey.getKeys(feature)) {
long value = get(recordKey);
long threshold = getProfileThreshold(recordKey);
long warnThreshold = getProfileWarnThreshold(recordKey);
ApiUsageStateValue tmpValue;
if (threshold == 0 || value < warnThreshold) {
tmpValue = ApiUsageStateValue.ENABLED;
} else if (value < threshold) {
tmpValue = ApiUsageStateValue.WARNING;
} else {
tmpValue = ApiUsageStateValue.DISABLED;
}
featureValue = ApiUsageStateValue.toMoreRestricted(featureValue, tmpValue);
}
return setFeatureValue(feature, featureValue) ? Pair.of(feature, featureValue) : null;
}
}

15
application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java

@ -30,7 +30,8 @@ import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.CustomerId;
@ -127,13 +128,16 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
public void createDefaultTenantProfiles() throws Exception { public void createDefaultTenantProfiles() throws Exception {
tenantProfileService.findOrCreateDefaultTenantProfile(TenantId.SYS_TENANT_ID); tenantProfileService.findOrCreateDefaultTenantProfile(TenantId.SYS_TENANT_ID);
TenantProfileData tenantProfileData = new TenantProfileData();
tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration());
TenantProfile isolatedTbCoreProfile = new TenantProfile(); TenantProfile isolatedTbCoreProfile = new TenantProfile();
isolatedTbCoreProfile.setDefault(false); isolatedTbCoreProfile.setDefault(false);
isolatedTbCoreProfile.setName("Isolated TB Core"); isolatedTbCoreProfile.setName("Isolated TB Core");
isolatedTbCoreProfile.setProfileData(new TenantProfileData());
isolatedTbCoreProfile.setDescription("Isolated TB Core tenant profile"); isolatedTbCoreProfile.setDescription("Isolated TB Core tenant profile");
isolatedTbCoreProfile.setIsolatedTbCore(true); isolatedTbCoreProfile.setIsolatedTbCore(true);
isolatedTbCoreProfile.setIsolatedTbRuleEngine(false); isolatedTbCoreProfile.setIsolatedTbRuleEngine(false);
isolatedTbCoreProfile.setProfileData(tenantProfileData);
try { try {
tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreProfile); tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreProfile);
} catch (DataValidationException e) { } catch (DataValidationException e) {
@ -143,10 +147,11 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
TenantProfile isolatedTbRuleEngineProfile = new TenantProfile(); TenantProfile isolatedTbRuleEngineProfile = new TenantProfile();
isolatedTbRuleEngineProfile.setDefault(false); isolatedTbRuleEngineProfile.setDefault(false);
isolatedTbRuleEngineProfile.setName("Isolated TB Rule Engine"); isolatedTbRuleEngineProfile.setName("Isolated TB Rule Engine");
isolatedTbRuleEngineProfile.setProfileData(new TenantProfileData());
isolatedTbRuleEngineProfile.setDescription("Isolated TB Rule Engine tenant profile"); isolatedTbRuleEngineProfile.setDescription("Isolated TB Rule Engine tenant profile");
isolatedTbRuleEngineProfile.setIsolatedTbCore(false); isolatedTbRuleEngineProfile.setIsolatedTbCore(false);
isolatedTbRuleEngineProfile.setIsolatedTbRuleEngine(true); isolatedTbRuleEngineProfile.setIsolatedTbRuleEngine(true);
isolatedTbRuleEngineProfile.setProfileData(tenantProfileData);
try { try {
tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbRuleEngineProfile); tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbRuleEngineProfile);
} catch (DataValidationException e) { } catch (DataValidationException e) {
@ -156,10 +161,11 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
TenantProfile isolatedTbCoreAndTbRuleEngineProfile = new TenantProfile(); TenantProfile isolatedTbCoreAndTbRuleEngineProfile = new TenantProfile();
isolatedTbCoreAndTbRuleEngineProfile.setDefault(false); isolatedTbCoreAndTbRuleEngineProfile.setDefault(false);
isolatedTbCoreAndTbRuleEngineProfile.setName("Isolated TB Core and TB Rule Engine"); isolatedTbCoreAndTbRuleEngineProfile.setName("Isolated TB Core and TB Rule Engine");
isolatedTbCoreAndTbRuleEngineProfile.setProfileData(new TenantProfileData());
isolatedTbCoreAndTbRuleEngineProfile.setDescription("Isolated TB Core and TB Rule Engine tenant profile"); isolatedTbCoreAndTbRuleEngineProfile.setDescription("Isolated TB Core and TB Rule Engine tenant profile");
isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbCore(true); isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbCore(true);
isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbRuleEngine(true); isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbRuleEngine(true);
isolatedTbCoreAndTbRuleEngineProfile.setProfileData(tenantProfileData);
try { try {
tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreAndTbRuleEngineProfile); tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreAndTbRuleEngineProfile);
} catch (DataValidationException e) { } catch (DataValidationException e) {
@ -173,6 +179,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
generalSettings.setKey("general"); generalSettings.setKey("general");
ObjectNode node = objectMapper.createObjectNode(); ObjectNode node = objectMapper.createObjectNode();
node.put("baseUrl", "http://localhost:8080"); node.put("baseUrl", "http://localhost:8080");
node.put("prohibitDifferentUrl", true);
generalSettings.setJsonValue(node); generalSettings.setJsonValue(node);
adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, generalSettings); adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, generalSettings);

24
application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java

@ -28,6 +28,7 @@ import org.thingsboard.server.dao.dashboard.DashboardService;
import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
import org.thingsboard.server.service.install.sql.SqlDbHelper; import org.thingsboard.server.service.install.sql.SqlDbHelper;
import java.nio.charset.Charset; import java.nio.charset.Charset;
@ -96,6 +97,9 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
@Autowired @Autowired
private DeviceProfileService deviceProfileService; private DeviceProfileService deviceProfileService;
@Autowired
private ApiUsageStateService apiUsageStateService;
@Override @Override
public void upgradeDatabase(String fromVersion) throws Exception { public void upgradeDatabase(String fromVersion) throws Exception {
@ -352,6 +356,22 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
} catch (Exception e) { } catch (Exception e) {
} }
try {
conn.createStatement().execute("CREATE TABLE IF NOT EXISTS api_usage_state (" +
" id uuid NOT NULL CONSTRAINT usage_record_pkey PRIMARY KEY," +
" created_time bigint NOT NULL," +
" tenant_id uuid," +
" entity_type varchar(32)," +
" entity_id uuid," +
" transport varchar(32)," +
" db_storage varchar(32)," +
" re_exec varchar(32)," +
" js_exec varchar(32)," +
" CONSTRAINT api_usage_state_unq_key UNIQUE (tenant_id, entity_id)\n" +
");");
} catch (Exception e) {
}
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.1.1", "schema_update_before.sql"); schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.1.1", "schema_update_before.sql");
loadSql(schemaUpdateFile, conn); loadSql(schemaUpdateFile, conn);
@ -367,6 +387,10 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
do { do {
pageData = tenantService.findTenants(pageLink); pageData = tenantService.findTenants(pageLink);
for (Tenant tenant : pageData.getData()) { for (Tenant tenant : pageData.getData()) {
try {
apiUsageStateService.createDefaultApiUsageState(tenant.getId());
} catch (Exception e) {
}
List<EntitySubtype> deviceTypes = deviceService.findDeviceTypesByTenantId(tenant.getId()).get(); List<EntitySubtype> deviceTypes = deviceService.findDeviceTypesByTenantId(tenant.getId()).get();
try { try {
deviceProfileService.createDefaultDeviceProfile(tenant.getId()); deviceProfileService.createDefaultDeviceProfile(tenant.getId());

56
application/src/main/java/org/thingsboard/server/service/profile/DefaultTbDeviceProfileCache.java

@ -26,11 +26,11 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.device.DeviceService;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
import java.util.function.BiConsumer;
import java.util.function.Consumer; import java.util.function.Consumer;
@Service @Service
@ -43,7 +43,8 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache {
private final ConcurrentMap<DeviceProfileId, DeviceProfile> deviceProfilesMap = new ConcurrentHashMap<>(); private final ConcurrentMap<DeviceProfileId, DeviceProfile> deviceProfilesMap = new ConcurrentHashMap<>();
private final ConcurrentMap<DeviceId, DeviceProfileId> devicesMap = new ConcurrentHashMap<>(); private final ConcurrentMap<DeviceId, DeviceProfileId> devicesMap = new ConcurrentHashMap<>();
private final ConcurrentMap<TenantId, ConcurrentMap<EntityId, Consumer<DeviceProfile>>> listeners = new ConcurrentHashMap<>(); private final ConcurrentMap<TenantId, ConcurrentMap<EntityId, Consumer<DeviceProfile>>> profileListeners = new ConcurrentHashMap<>();
private final ConcurrentMap<TenantId, ConcurrentMap<EntityId, BiConsumer<DeviceId, DeviceProfile>>> deviceProfileListeners = new ConcurrentHashMap<>();
public DefaultTbDeviceProfileCache(DeviceProfileService deviceProfileService, DeviceService deviceService) { public DefaultTbDeviceProfileCache(DeviceProfileService deviceProfileService, DeviceService deviceService) {
this.deviceProfileService = deviceProfileService; this.deviceProfileService = deviceProfileService;
@ -87,33 +88,37 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache {
return get(tenantId, profileId); return get(tenantId, profileId);
} }
@Override
public void put(DeviceProfile profile) {
if (profile.getId() != null) {
deviceProfilesMap.put(profile.getId(), profile);
log.debug("[{}] pushed device profile to cache: {}", profile.getId(), profile);
notifyListeners(profile);
}
}
@Override @Override
public void evict(TenantId tenantId, DeviceProfileId profileId) { public void evict(TenantId tenantId, DeviceProfileId profileId) {
DeviceProfile oldProfile = deviceProfilesMap.remove(profileId); DeviceProfile oldProfile = deviceProfilesMap.remove(profileId);
log.debug("[{}] evict device profile from cache: {}", profileId, oldProfile); log.debug("[{}] evict device profile from cache: {}", profileId, oldProfile);
DeviceProfile newProfile = get(tenantId, profileId); DeviceProfile newProfile = get(tenantId, profileId);
if (newProfile != null) { if (newProfile != null) {
notifyListeners(newProfile); notifyProfileListeners(newProfile);
} }
} }
@Override @Override
public void evict(DeviceId deviceId) { public void evict(TenantId tenantId, DeviceId deviceId) {
devicesMap.remove(deviceId); DeviceProfileId old = devicesMap.remove(deviceId);
if (old != null) {
DeviceProfile newProfile = get(tenantId, deviceId);
if (newProfile == null || !old.equals(newProfile.getId())) {
notifyDeviceListeners(tenantId, deviceId, newProfile);
}
}
} }
@Override @Override
public void addListener(TenantId tenantId, EntityId listenerId, Consumer<DeviceProfile> listener) { public void addListener(TenantId tenantId, EntityId listenerId,
listeners.computeIfAbsent(tenantId, id -> new ConcurrentHashMap<>()).put(listenerId, listener); Consumer<DeviceProfile> profileListener,
BiConsumer<DeviceId, DeviceProfile> deviceListener) {
if (profileListener != null) {
profileListeners.computeIfAbsent(tenantId, id -> new ConcurrentHashMap<>()).put(listenerId, profileListener);
}
if (deviceListener != null) {
deviceProfileListeners.computeIfAbsent(tenantId, id -> new ConcurrentHashMap<>()).put(listenerId, deviceListener);
}
} }
@Override @Override
@ -128,17 +133,30 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache {
@Override @Override
public void removeListener(TenantId tenantId, EntityId listenerId) { public void removeListener(TenantId tenantId, EntityId listenerId) {
ConcurrentMap<EntityId, Consumer<DeviceProfile>> tenantListeners = listeners.get(tenantId); ConcurrentMap<EntityId, Consumer<DeviceProfile>> tenantListeners = profileListeners.get(tenantId);
if (tenantListeners != null) { if (tenantListeners != null) {
tenantListeners.remove(listenerId); tenantListeners.remove(listenerId);
} }
ConcurrentMap<EntityId, BiConsumer<DeviceId, DeviceProfile>> deviceListeners = deviceProfileListeners.get(tenantId);
if (deviceListeners != null) {
deviceListeners.remove(listenerId);
}
} }
private void notifyListeners(DeviceProfile profile) { private void notifyProfileListeners(DeviceProfile profile) {
ConcurrentMap<EntityId, Consumer<DeviceProfile>> tenantListeners = listeners.get(profile.getTenantId()); ConcurrentMap<EntityId, Consumer<DeviceProfile>> tenantListeners = profileListeners.get(profile.getTenantId());
if (tenantListeners != null) { if (tenantListeners != null) {
tenantListeners.forEach((id, listener) -> listener.accept(profile)); tenantListeners.forEach((id, listener) -> listener.accept(profile));
} }
} }
private void notifyDeviceListeners(TenantId tenantId, DeviceId deviceId, DeviceProfile profile) {
if (profile != null) {
ConcurrentMap<EntityId, BiConsumer<DeviceId, DeviceProfile>> tenantListeners = deviceProfileListeners.get(tenantId);
if (tenantListeners != null) {
tenantListeners.forEach((id, listener) -> listener.accept(deviceId, profile));
}
}
}
} }

4
application/src/main/java/org/thingsboard/server/service/profile/TbDeviceProfileCache.java

@ -23,11 +23,9 @@ import org.thingsboard.server.common.data.id.TenantId;
public interface TbDeviceProfileCache extends RuleEngineDeviceProfileCache { public interface TbDeviceProfileCache extends RuleEngineDeviceProfileCache {
void put(DeviceProfile profile);
void evict(TenantId tenantId, DeviceProfileId id); void evict(TenantId tenantId, DeviceProfileId id);
void evict(DeviceId id); void evict(TenantId tenantId, DeviceId id);
DeviceProfile find(DeviceProfileId deviceProfileId); DeviceProfile find(DeviceProfileId deviceProfileId);

36
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java

@ -21,6 +21,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasName;
@ -47,6 +48,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotifica
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueCallback;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.MultipleTbQueueCallbackWrapper;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
@ -156,8 +158,16 @@ public class DefaultTbClusterService implements TbClusterService {
private TbMsg transformMsg(TbMsg tbMsg, DeviceProfile deviceProfile) { private TbMsg transformMsg(TbMsg tbMsg, DeviceProfile deviceProfile) {
if (deviceProfile != null) { if (deviceProfile != null) {
RuleChainId targetRuleChainId = deviceProfile.getDefaultRuleChainId(); RuleChainId targetRuleChainId = deviceProfile.getDefaultRuleChainId();
if (targetRuleChainId != null && !targetRuleChainId.equals(tbMsg.getRuleChainId())) { String targetQueueName = deviceProfile.getDefaultQueueName();
boolean isRuleChainTransform = targetRuleChainId != null && !targetRuleChainId.equals(tbMsg.getRuleChainId());
boolean isQueueTransform = targetQueueName != null && !targetQueueName.equals(tbMsg.getQueueName());
if (isRuleChainTransform && isQueueTransform) {
tbMsg = TbMsg.transformMsg(tbMsg, targetRuleChainId, targetQueueName);
} else if (isRuleChainTransform) {
tbMsg = TbMsg.transformMsg(tbMsg, targetRuleChainId); tbMsg = TbMsg.transformMsg(tbMsg, targetRuleChainId);
} else if (isQueueTransform) {
tbMsg = TbMsg.transformMsg(tbMsg, targetQueueName);
} }
} }
return tbMsg; return tbMsg;
@ -206,6 +216,12 @@ public class DefaultTbClusterService implements TbClusterService {
onEntityChange(TenantId.SYS_TENANT_ID, tenant.getId(), tenant, callback); onEntityChange(TenantId.SYS_TENANT_ID, tenant.getId(), tenant, callback);
} }
@Override
public void onApiStateChange(ApiUsageState apiUsageState, TbQueueCallback callback) {
onEntityChange(apiUsageState.getTenantId(), apiUsageState.getId(), apiUsageState, callback);
broadcast(new ComponentLifecycleMsg(apiUsageState.getTenantId(), apiUsageState.getId(), ComponentLifecycleEvent.UPDATED));
}
@Override @Override
public void onDeviceProfileDelete(DeviceProfile entity, TbQueueCallback callback) { public void onDeviceProfileDelete(DeviceProfile entity, TbQueueCallback callback) {
onEntityDelete(entity.getTenantId(), entity.getId(), entity.getName(), callback); onEntityDelete(entity.getTenantId(), entity.getId(), entity.getName(), callback);
@ -221,13 +237,14 @@ public class DefaultTbClusterService implements TbClusterService {
onEntityDelete(TenantId.SYS_TENANT_ID, entity.getId(), entity.getName(), callback); onEntityDelete(TenantId.SYS_TENANT_ID, entity.getId(), entity.getName(), callback);
} }
public <T extends HasName> void onEntityChange(TenantId tenantId, EntityId entityid, T entity, TbQueueCallback callback) { public <T> void onEntityChange(TenantId tenantId, EntityId entityid, T entity, TbQueueCallback callback) {
log.trace("[{}][{}][{}] Processing [{}] change event", tenantId, entityid.getEntityType(), entityid.getId(), entity.getName()); String entityName = (entity instanceof HasName) ? ((HasName) entity).getName() : entity.getClass().getName();
log.trace("[{}][{}][{}] Processing [{}] change event", tenantId, entityid.getEntityType(), entityid.getId(), entityName);
TransportProtos.EntityUpdateMsg entityUpdateMsg = TransportProtos.EntityUpdateMsg.newBuilder() TransportProtos.EntityUpdateMsg entityUpdateMsg = TransportProtos.EntityUpdateMsg.newBuilder()
.setEntityType(entityid.getEntityType().name()) .setEntityType(entityid.getEntityType().name())
.setData(ByteString.copyFrom(encodingService.encode(entity))).build(); .setData(ByteString.copyFrom(encodingService.encode(entity))).build();
ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setEntityUpdateMsg(entityUpdateMsg).build(); ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setEntityUpdateMsg(entityUpdateMsg).build();
broadcast(transportMsg); broadcast(transportMsg, callback);
} }
private void onEntityDelete(TenantId tenantId, EntityId entityId, String name, TbQueueCallback callback) { private void onEntityDelete(TenantId tenantId, EntityId entityId, String name, TbQueueCallback callback) {
@ -238,15 +255,16 @@ public class DefaultTbClusterService implements TbClusterService {
.setEntityIdLSB(entityId.getId().getLeastSignificantBits()) .setEntityIdLSB(entityId.getId().getLeastSignificantBits())
.build(); .build();
ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setEntityDeleteMsg(entityDeleteMsg).build(); ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setEntityDeleteMsg(entityDeleteMsg).build();
broadcast(transportMsg); broadcast(transportMsg, callback);
} }
private void broadcast(ToTransportMsg transportMsg) { private void broadcast(ToTransportMsg transportMsg, TbQueueCallback callback) {
TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> toTransportNfProducer = producerProvider.getTransportNotificationsMsgProducer(); TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> toTransportNfProducer = producerProvider.getTransportNotificationsMsgProducer();
Set<String> tbTransportServices = partitionService.getAllServiceIds(ServiceType.TB_TRANSPORT); Set<String> tbTransportServices = partitionService.getAllServiceIds(ServiceType.TB_TRANSPORT);
TbQueueCallback proxyCallback = callback != null ? new MultipleTbQueueCallbackWrapper(tbTransportServices.size(), callback) : null;
for (String transportServiceId : tbTransportServices) { for (String transportServiceId : tbTransportServices) {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, transportServiceId); TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, transportServiceId);
toTransportNfProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), null); toTransportNfProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), proxyCallback);
toTransportNfs.incrementAndGet(); toTransportNfs.incrementAndGet();
} }
} }
@ -256,7 +274,9 @@ public class DefaultTbClusterService implements TbClusterService {
TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer();
Set<String> tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE)); Set<String> tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE));
if (msg.getEntityId().getEntityType().equals(EntityType.TENANT) if (msg.getEntityId().getEntityType().equals(EntityType.TENANT)
|| msg.getEntityId().getEntityType().equals(EntityType.DEVICE_PROFILE)) { || msg.getEntityId().getEntityType().equals(EntityType.TENANT_PROFILE)
|| msg.getEntityId().getEntityType().equals(EntityType.DEVICE_PROFILE)
|| msg.getEntityId().getEntityType().equals(EntityType.API_USAGE_STATE)) {
TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer();
Set<String> tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); Set<String> tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE);
for (String serviceId : tbCoreServices) { for (String serviceId : tbCoreServices) {

113
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -17,41 +17,45 @@ package org.thingsboard.server.service.queue;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.core.annotation.Order;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.rule.engine.api.RpcError; import org.thingsboard.rule.engine.api.RpcError;
import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.Alarm;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.MsgType;
import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.dao.util.mapping.JacksonUtil; import org.thingsboard.server.dao.util.mapping.JacksonUtil;
import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto;
import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto;
import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto;
import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeDeleteProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmUpdateProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmDeleteProto; import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmDeleteProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmUpdateProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeDeleteProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto; import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto; import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService;
@ -70,6 +74,8 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch; import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.function.Function; import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -88,32 +94,57 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
private final TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> mainConsumer; private final TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> mainConsumer;
private final DeviceStateService stateService; private final DeviceStateService stateService;
private final TbApiUsageStateService statsService;
private final TbLocalSubscriptionService localSubscriptionService; private final TbLocalSubscriptionService localSubscriptionService;
private final SubscriptionManagerService subscriptionManagerService; private final SubscriptionManagerService subscriptionManagerService;
private final TbCoreDeviceRpcService tbCoreDeviceRpcService; private final TbCoreDeviceRpcService tbCoreDeviceRpcService;
private final TbCoreConsumerStats stats; private final TbCoreConsumerStats stats;
protected final TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> usageStatsConsumer;
protected volatile ExecutorService usageStatsExecutor;
public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory,
DeviceStateService stateService, TbLocalSubscriptionService localSubscriptionService, ActorSystemContext actorContext,
SubscriptionManagerService subscriptionManagerService, DataDecodingEncodingService encodingService, DeviceStateService stateService,
TbCoreDeviceRpcService tbCoreDeviceRpcService, StatsFactory statsFactory, TbDeviceProfileCache deviceProfileCache) { TbLocalSubscriptionService localSubscriptionService,
super(actorContext, encodingService, deviceProfileCache, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer()); SubscriptionManagerService subscriptionManagerService,
DataDecodingEncodingService encodingService,
TbCoreDeviceRpcService tbCoreDeviceRpcService,
StatsFactory statsFactory,
TbDeviceProfileCache deviceProfileCache,
TbApiUsageStateService statsService,
TbTenantProfileCache tenantProfileCache,
TbApiUsageStateService apiUsageStateService) {
super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, apiUsageStateService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer());
this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer(); this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer();
this.usageStatsConsumer = tbCoreQueueFactory.createToUsageStatsServiceMsgConsumer();
this.stateService = stateService; this.stateService = stateService;
this.localSubscriptionService = localSubscriptionService; this.localSubscriptionService = localSubscriptionService;
this.subscriptionManagerService = subscriptionManagerService; this.subscriptionManagerService = subscriptionManagerService;
this.tbCoreDeviceRpcService = tbCoreDeviceRpcService; this.tbCoreDeviceRpcService = tbCoreDeviceRpcService;
this.stats = new TbCoreConsumerStats(statsFactory); this.stats = new TbCoreConsumerStats(statsFactory);
this.statsService = statsService;
} }
@PostConstruct @PostConstruct
public void init() { public void init() {
super.init("tb-core-consumer", "tb-core-notifications-consumer"); super.init("tb-core-consumer", "tb-core-notifications-consumer");
this.usageStatsExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-core-usage-stats-consumer"));
} }
@PreDestroy @PreDestroy
public void destroy() { public void destroy() {
super.destroy(); super.destroy();
if (usageStatsExecutor != null) {
usageStatsExecutor.shutdownNow();
}
}
@EventListener(ApplicationReadyEvent.class)
@Order(value = 2)
public void onApplicationEvent(ApplicationReadyEvent event) {
super.onApplicationEvent(event);
launchUsageStatsConsumer();
} }
@Override @Override
@ -121,6 +152,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
if (partitionChangeEvent.getServiceType().equals(getServiceType())) { if (partitionChangeEvent.getServiceType().equals(getServiceType())) {
log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions());
this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); this.mainConsumer.subscribe(partitionChangeEvent.getPartitions());
this.usageStatsConsumer.subscribe(
partitionChangeEvent
.getPartitions()
.stream()
.map(tpi -> tpi.newByTopic(usageStatsConsumer.getTopic()))
.collect(Collectors.toSet()));
} }
} }
@ -223,6 +260,53 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
} }
} }
private void launchUsageStatsConsumer() {
usageStatsExecutor.submit(() -> {
while (!stopped) {
try {
List<TbProtoQueueMsg<ToUsageStatsServiceMsg>> msgs = usageStatsConsumer.poll(getNotificationPollDuration());
if (msgs.isEmpty()) {
continue;
}
ConcurrentMap<UUID, TbProtoQueueMsg<ToUsageStatsServiceMsg>> pendingMap = msgs.stream().collect(
Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity()));
CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
TbPackProcessingContext<TbProtoQueueMsg<ToUsageStatsServiceMsg>> ctx = new TbPackProcessingContext<>(
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
pendingMap.forEach((id, msg) -> {
log.trace("[{}] Creating usage stats callback for message: {}", id, msg.getValue());
TbCallback callback = new TbPackCallback<>(id, ctx);
try {
handleUsageStats(msg, callback);
} catch (Throwable e) {
log.warn("[{}] Failed to process usge stats: {}", id, msg, e);
callback.onFailure(e);
}
});
if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) {
ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process usage stats: {}", id, msg.getValue()));
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process usage stats: {}", id, msg.getValue()));
}
usageStatsConsumer.commit();
} catch (Exception e) {
if (!stopped) {
log.warn("Failed to obtain usage stats from queue.", e);
try {
Thread.sleep(getNotificationPollDuration());
} catch (InterruptedException e2) {
log.trace("Failed to wait until the server has capacity to handle new usage stats", e2);
}
}
}
}
log.info("TB Usage Stats Consumer stopped.");
});
}
private void handleUsageStats(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback) {
statsService.process(msg, callback);
}
private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbCallback callback) { private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbCallback callback) {
RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null;
FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()) FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB())
@ -321,6 +405,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
if (mainConsumer != null) { if (mainConsumer != null) {
mainConsumer.unsubscribe(); mainConsumer.unsubscribe();
} }
if (usageStatsConsumer != null) {
usageStatsConsumer.unsubscribe();
}
} }
} }

48
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@ -15,7 +15,6 @@
*/ */
package org.thingsboard.server.service.queue; package org.thingsboard.server.service.queue;
import com.google.protobuf.ByteString;
import com.google.protobuf.ProtocolStringList; import com.google.protobuf.ProtocolStringList;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
@ -24,10 +23,16 @@ import org.springframework.stereotype.Service;
import org.thingsboard.rule.engine.api.RpcError; import org.thingsboard.rule.engine.api.RpcError;
import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.queue.*; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg;
import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
import org.thingsboard.server.common.msg.queue.ServiceQueue;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
@ -38,17 +43,33 @@ import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.queue.util.TbRuleEngineComponent;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.service.queue.processing.*; import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory;
import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService;
import org.thingsboard.server.service.stats.RuleEngineStatisticsService; import org.thingsboard.server.service.stats.RuleEngineStatisticsService;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy; import javax.annotation.PreDestroy;
import java.util.*; import java.util.Collections;
import java.util.concurrent.*; import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@Service @Service
@TbRuleEngineComponent @TbRuleEngineComponent
@ -79,11 +100,16 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory,
TbRuleEngineSubmitStrategyFactory submitStrategyFactory, TbRuleEngineSubmitStrategyFactory submitStrategyFactory,
TbQueueRuleEngineSettings ruleEngineSettings, TbQueueRuleEngineSettings ruleEngineSettings,
TbRuleEngineQueueFactory tbRuleEngineQueueFactory, RuleEngineStatisticsService statisticsService, TbRuleEngineQueueFactory tbRuleEngineQueueFactory,
ActorSystemContext actorContext, DataDecodingEncodingService encodingService, RuleEngineStatisticsService statisticsService,
ActorSystemContext actorContext,
DataDecodingEncodingService encodingService,
TbRuleEngineDeviceRpcService tbDeviceRpcService, TbRuleEngineDeviceRpcService tbDeviceRpcService,
StatsFactory statsFactory, TbDeviceProfileCache deviceProfileCache) { StatsFactory statsFactory,
super(actorContext, encodingService, deviceProfileCache, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer()); TbDeviceProfileCache deviceProfileCache,
TbTenantProfileCache tenantProfileCache,
TbApiUsageStateService apiUsageStateService) {
super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, apiUsageStateService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer());
this.statisticsService = statisticsService; this.statisticsService = statisticsService;
this.ruleEngineSettings = ruleEngineSettings; this.ruleEngineSettings = ruleEngineSettings;
this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory;

3
application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java

@ -21,11 +21,10 @@ import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.queue.discovery.TenantRoutingInfo; import org.thingsboard.server.queue.discovery.TenantRoutingInfo;
import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; import org.thingsboard.server.queue.discovery.TenantRoutingInfoService;
import org.thingsboard.server.service.profile.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
@Slf4j @Slf4j
@Service @Service

3
application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java

@ -16,6 +16,7 @@
package org.thingsboard.server.service.queue; package org.thingsboard.server.service.queue;
import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.TenantProfile;
@ -63,4 +64,6 @@ public interface TbClusterService {
void onTenantChange(Tenant tenant, TbQueueCallback callback); void onTenantChange(Tenant tenant, TbQueueCallback callback);
void onTenantDelete(Tenant tenant, TbQueueCallback callback); void onTenantDelete(Tenant tenant, TbQueueCallback callback);
void onApiStateChange(ApiUsageState apiUsageState, TbQueueCallback callback);
} }

31
application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java

@ -20,11 +20,14 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.ApplicationListener; import org.springframework.context.ApplicationListener;
import org.springframework.context.event.EventListener; import org.springframework.context.event.EventListener;
import org.springframework.core.annotation.Order;
import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.TenantProfileId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
@ -33,7 +36,9 @@ import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.service.queue.TbPackCallback; import org.thingsboard.server.service.queue.TbPackCallback;
import org.thingsboard.server.service.queue.TbPackProcessingContext; import org.thingsboard.server.service.queue.TbPackProcessingContext;
@ -59,15 +64,19 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
protected final ActorSystemContext actorContext; protected final ActorSystemContext actorContext;
protected final DataDecodingEncodingService encodingService; protected final DataDecodingEncodingService encodingService;
protected final TbTenantProfileCache tenantProfileCache;
protected final TbDeviceProfileCache deviceProfileCache; protected final TbDeviceProfileCache deviceProfileCache;
protected final TbApiUsageStateService apiUsageStateService;
protected final TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer; protected final TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer;
public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService,
TbDeviceProfileCache deviceProfileCache, TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer) { TbTenantProfileCache tenantProfileCache, TbDeviceProfileCache deviceProfileCache, TbApiUsageStateService apiUsageStateService, TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer) {
this.actorContext = actorContext; this.actorContext = actorContext;
this.encodingService = encodingService; this.encodingService = encodingService;
this.tenantProfileCache = tenantProfileCache;
this.deviceProfileCache = deviceProfileCache; this.deviceProfileCache = deviceProfileCache;
this.apiUsageStateService = apiUsageStateService;
this.nfConsumer = nfConsumer; this.nfConsumer = nfConsumer;
} }
@ -77,6 +86,7 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
} }
@EventListener(ApplicationReadyEvent.class) @EventListener(ApplicationReadyEvent.class)
@Order(value = 2)
public void onApplicationEvent(ApplicationReadyEvent event) { public void onApplicationEvent(ApplicationReadyEvent event) {
log.info("Subscribing to notifications: {}", nfConsumer.getTopic()); log.info("Subscribing to notifications: {}", nfConsumer.getTopic());
this.nfConsumer.subscribe(); this.nfConsumer.subscribe();
@ -143,10 +153,25 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
TbActorMsg actorMsg = actorMsgOpt.get(); TbActorMsg actorMsg = actorMsgOpt.get();
if (actorMsg instanceof ComponentLifecycleMsg) { if (actorMsg instanceof ComponentLifecycleMsg) {
ComponentLifecycleMsg componentLifecycleMsg = (ComponentLifecycleMsg) actorMsg; ComponentLifecycleMsg componentLifecycleMsg = (ComponentLifecycleMsg) actorMsg;
if (EntityType.DEVICE_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { log.info("[{}][{}][{}] Received Lifecycle event: {}", componentLifecycleMsg.getTenantId(), componentLifecycleMsg.getEntityId().getEntityType(),
componentLifecycleMsg.getEntityId(), componentLifecycleMsg.getEvent());
if (EntityType.TENANT_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
TenantProfileId tenantProfileId = new TenantProfileId(componentLifecycleMsg.getEntityId().getId());
tenantProfileCache.evict(tenantProfileId);
if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.UPDATED)) {
apiUsageStateService.onTenantProfileUpdate(tenantProfileId);
}
} else if (EntityType.TENANT.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
tenantProfileCache.evict(componentLifecycleMsg.getTenantId());
if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.UPDATED)) {
apiUsageStateService.onTenantUpdate(componentLifecycleMsg.getTenantId());
}
} else if (EntityType.DEVICE_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceProfileId(componentLifecycleMsg.getEntityId().getId())); deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceProfileId(componentLifecycleMsg.getEntityId().getId()));
} else if (EntityType.DEVICE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { } else if (EntityType.DEVICE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
deviceProfileCache.evict(new DeviceId(componentLifecycleMsg.getEntityId().getId())); deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceId(componentLifecycleMsg.getEntityId().getId()));
} else if (EntityType.API_USAGE_STATE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
apiUsageStateService.onApiUsageStateUpdate(componentLifecycleMsg.getTenantId());
} }
} }
log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg); log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg);

64
application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java

@ -19,6 +19,10 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import java.util.Map; import java.util.Map;
import java.util.UUID; import java.util.UUID;
@ -33,9 +37,16 @@ import java.util.concurrent.atomic.AtomicInteger;
@Slf4j @Slf4j
public abstract class AbstractJsInvokeService implements JsInvokeService { public abstract class AbstractJsInvokeService implements JsInvokeService {
private final TbApiUsageStateService apiUsageStateService;
private final TbApiUsageClient apiUsageClient;
protected ScheduledExecutorService timeoutExecutorService; protected ScheduledExecutorService timeoutExecutorService;
protected Map<UUID, String> scriptIdToNameMap = new ConcurrentHashMap<>(); protected Map<UUID, String> scriptIdToNameMap = new ConcurrentHashMap<>();
protected Map<UUID, BlackListInfo> blackListedFunctions = new ConcurrentHashMap<>(); protected Map<UUID, DisableListInfo> disabledFunctions = new ConcurrentHashMap<>();
protected AbstractJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient) {
this.apiUsageStateService = apiUsageStateService;
this.apiUsageClient = apiUsageClient;
}
public void init(long maxRequestsTimeout) { public void init(long maxRequestsTimeout) {
if (maxRequestsTimeout > 0) { if (maxRequestsTimeout > 0) {
@ -50,24 +61,33 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
} }
@Override @Override
public ListenableFuture<UUID> eval(JsScriptType scriptType, String scriptBody, String... argNames) { public ListenableFuture<UUID> eval(TenantId tenantId, JsScriptType scriptType, String scriptBody, String... argNames) {
UUID scriptId = UUID.randomUUID(); if (apiUsageStateService.getApiUsageState(tenantId).isJsExecEnabled()) {
String functionName = "invokeInternal_" + scriptId.toString().replace('-', '_'); UUID scriptId = UUID.randomUUID();
String jsScript = generateJsScript(scriptType, functionName, scriptBody, argNames); String functionName = "invokeInternal_" + scriptId.toString().replace('-', '_');
return doEval(scriptId, functionName, jsScript); String jsScript = generateJsScript(scriptType, functionName, scriptBody, argNames);
return doEval(scriptId, functionName, jsScript);
} else {
return Futures.immediateFailedFuture(new RuntimeException("JS Execution is disabled due to API limits!"));
}
} }
@Override @Override
public ListenableFuture<Object> invokeFunction(UUID scriptId, Object... args) { public ListenableFuture<Object> invokeFunction(TenantId tenantId, UUID scriptId, Object... args) {
String functionName = scriptIdToNameMap.get(scriptId); if (apiUsageStateService.getApiUsageState(tenantId).isJsExecEnabled()) {
if (functionName == null) { String functionName = scriptIdToNameMap.get(scriptId);
return Futures.immediateFailedFuture(new RuntimeException("No compiled script found for scriptId: [" + scriptId + "]!")); if (functionName == null) {
} return Futures.immediateFailedFuture(new RuntimeException("No compiled script found for scriptId: [" + scriptId + "]!"));
if (!isBlackListed(scriptId)) { }
return doInvokeFunction(scriptId, functionName, args); if (!isDisabled(scriptId)) {
apiUsageClient.report(tenantId, ApiUsageRecordKey.JS_EXEC_COUNT, 1);
return doInvokeFunction(scriptId, functionName, args);
} else {
return Futures.immediateFailedFuture(
new RuntimeException("Script invocation is blocked due to maximum error count " + getMaxErrors() + "!"));
}
} else { } else {
return Futures.immediateFailedFuture( return Futures.immediateFailedFuture(new RuntimeException("JS Execution is disabled due to API limits!"));
new RuntimeException("Script is blacklisted due to maximum error count " + getMaxErrors() + "!"));
} }
} }
@ -77,7 +97,7 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
if (functionName != null) { if (functionName != null) {
try { try {
scriptIdToNameMap.remove(scriptId); scriptIdToNameMap.remove(scriptId);
blackListedFunctions.remove(scriptId); disabledFunctions.remove(scriptId);
doRelease(scriptId, functionName); doRelease(scriptId, functionName);
} catch (Exception e) { } catch (Exception e) {
return Futures.immediateFailedFuture(e); return Futures.immediateFailedFuture(e);
@ -97,7 +117,7 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
protected abstract long getMaxBlacklistDuration(); protected abstract long getMaxBlacklistDuration();
protected void onScriptExecutionError(UUID scriptId) { protected void onScriptExecutionError(UUID scriptId) {
blackListedFunctions.computeIfAbsent(scriptId, key -> new BlackListInfo()).incrementAndGet(); disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()).incrementAndGet();
} }
private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) { private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) {
@ -107,11 +127,11 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); throw new RuntimeException("No script factory implemented for scriptType: " + scriptType);
} }
private boolean isBlackListed(UUID scriptId) { private boolean isDisabled(UUID scriptId) {
BlackListInfo errorCount = blackListedFunctions.get(scriptId); DisableListInfo errorCount = disabledFunctions.get(scriptId);
if (errorCount != null) { if (errorCount != null) {
if (errorCount.getExpirationTime() <= System.currentTimeMillis()) { if (errorCount.getExpirationTime() <= System.currentTimeMillis()) {
blackListedFunctions.remove(scriptId); disabledFunctions.remove(scriptId);
return false; return false;
} else { } else {
return errorCount.get() >= getMaxErrors(); return errorCount.get() >= getMaxErrors();
@ -121,11 +141,11 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
} }
} }
private class BlackListInfo { private class DisableListInfo {
private final AtomicInteger counter; private final AtomicInteger counter;
private long expirationTime; private long expirationTime;
private BlackListInfo() { private DisableListInfo() {
this.counter = new AtomicInteger(0); this.counter = new AtomicInteger(0);
} }

13
application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java

@ -24,10 +24,10 @@ import delight.nashornsandbox.NashornSandboxes;
import jdk.nashorn.api.scripting.NashornScriptEngineFactory; import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
import lombok.Getter; import lombok.Getter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy; import javax.annotation.PreDestroy;
@ -38,7 +38,6 @@ import java.util.UUID;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
@ -57,9 +56,8 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
private final FutureCallback<UUID> evalCallback = new JsStatCallback<>(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs); private final FutureCallback<UUID> evalCallback = new JsStatCallback<>(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs);
private final FutureCallback<Object> invokeCallback = new JsStatCallback<>(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs); private final FutureCallback<Object> invokeCallback = new JsStatCallback<>(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs);
@Autowired
@Getter @Getter
private JsExecutorService jsExecutor; private final JsExecutorService jsExecutor;
@Value("${js.local.max_requests_timeout:0}") @Value("${js.local.max_requests_timeout:0}")
private long maxRequestsTimeout; private long maxRequestsTimeout;
@ -67,6 +65,11 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
@Value("${js.local.stats.enabled:false}") @Value("${js.local.stats.enabled:false}")
private boolean statsEnabled; private boolean statsEnabled;
public AbstractNashornJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient, JsExecutorService jsExecutor) {
super(apiUsageStateService, apiUsageClient);
this.jsExecutor = jsExecutor;
}
@Scheduled(fixedDelayString = "${js.local.stats.print_interval_ms:10000}") @Scheduled(fixedDelayString = "${js.local.stats.print_interval_ms:10000}")
public void printStats() { public void printStats() {
if (statsEnabled) { if (statsEnabled) {

2
application/src/main/java/org/thingsboard/server/service/script/JsExecutorService.java

@ -27,7 +27,7 @@ public class JsExecutorService extends AbstractListeningExecutor {
@Override @Override
protected int getThreadPollSize() { protected int getThreadPollSize() {
return jsExecutorThreadPoolSize; return Math.max(jsExecutorThreadPoolSize, 1);
} }
} }

5
application/src/main/java/org/thingsboard/server/service/script/JsInvokeService.java

@ -17,14 +17,15 @@ package org.thingsboard.server.service.script;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import java.util.UUID; import java.util.UUID;
public interface JsInvokeService { public interface JsInvokeService {
ListenableFuture<UUID> eval(JsScriptType scriptType, String scriptBody, String... argNames); ListenableFuture<UUID> eval(TenantId tenantId, JsScriptType scriptType, String scriptBody, String... argNames);
ListenableFuture<Object> invokeFunction(UUID scriptId, Object... args); ListenableFuture<Object> invokeFunction(TenantId tenantId, UUID scriptId, Object... args);
ListenableFuture<Void> release(UUID scriptId); ListenableFuture<Void> release(UUID scriptId);

6
application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java

@ -19,6 +19,8 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -42,6 +44,10 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService {
@Value("${js.local.max_black_list_duration_sec:60}") @Value("${js.local.max_black_list_duration_sec:60}")
private int maxBlackListDurationSec; private int maxBlackListDurationSec;
public NashornJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient, JsExecutorService jsExecutor) {
super(apiUsageStateService, apiUsageClient, jsExecutor);
}
@Override @Override
protected boolean useJsSandbox() { protected boolean useJsSandbox() {
return useJsSandbox; return useJsSandbox;

6
application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java

@ -30,6 +30,8 @@ import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
@ -68,6 +70,10 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
private final AtomicInteger queueFailedMsgs = new AtomicInteger(0); private final AtomicInteger queueFailedMsgs = new AtomicInteger(0);
private final AtomicInteger queueTimeoutMsgs = new AtomicInteger(0); private final AtomicInteger queueTimeoutMsgs = new AtomicInteger(0);
public RemoteJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient) {
super(apiUsageStateService, apiUsageClient);
}
@Scheduled(fixedDelayString = "${js.remote.stats.print_interval_ms}") @Scheduled(fixedDelayString = "${js.remote.stats.print_interval_ms}")
public void printStats() { public void printStats() {
if (statsEnabled) { if (statsEnabled) {

11
application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java

@ -25,6 +25,7 @@ import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.TbMsgMetaData;
@ -43,13 +44,15 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
private final JsInvokeService sandboxService; private final JsInvokeService sandboxService;
private final UUID scriptId; private final UUID scriptId;
private final TenantId tenantId;
private final EntityId entityId; private final EntityId entityId;
public RuleNodeJsScriptEngine(JsInvokeService sandboxService, EntityId entityId, String script, String... argNames) { public RuleNodeJsScriptEngine(TenantId tenantId, JsInvokeService sandboxService, EntityId entityId, String script, String... argNames) {
this.tenantId = tenantId;
this.sandboxService = sandboxService; this.sandboxService = sandboxService;
this.entityId = entityId; this.entityId = entityId;
try { try {
this.scriptId = this.sandboxService.eval(JsScriptType.RULE_NODE_SCRIPT, script, argNames).get(); this.scriptId = this.sandboxService.eval(tenantId, JsScriptType.RULE_NODE_SCRIPT, script, argNames).get();
} catch (Exception e) { } catch (Exception e) {
Throwable t = e; Throwable t = e;
if (e instanceof ExecutionException) { if (e instanceof ExecutionException) {
@ -203,7 +206,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
private JsonNode executeScript(TbMsg msg) throws ScriptException { private JsonNode executeScript(TbMsg msg) throws ScriptException {
try { try {
String[] inArgs = prepareArgs(msg); String[] inArgs = prepareArgs(msg);
String eval = sandboxService.invokeFunction(this.scriptId, inArgs[0], inArgs[1], inArgs[2]).get().toString(); String eval = sandboxService.invokeFunction(tenantId, this.scriptId, inArgs[0], inArgs[1], inArgs[2]).get().toString();
return mapper.readTree(eval); return mapper.readTree(eval);
} catch (ExecutionException e) { } catch (ExecutionException e) {
if (e.getCause() instanceof ScriptException) { if (e.getCause() instanceof ScriptException) {
@ -220,7 +223,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
private ListenableFuture<JsonNode> executeScriptAsync(TbMsg msg) { private ListenableFuture<JsonNode> executeScriptAsync(TbMsg msg) {
String[] inArgs = prepareArgs(msg); String[] inArgs = prepareArgs(msg);
return Futures.transformAsync(sandboxService.invokeFunction(this.scriptId, inArgs[0], inArgs[1], inArgs[2]), return Futures.transformAsync(sandboxService.invokeFunction(tenantId, this.scriptId, inArgs[0], inArgs[1], inArgs[2]),
o -> { o -> {
try { try {
return Futures.immediateFuture(mapper.readTree(o.toString())); return Futures.immediateFuture(mapper.readTree(o.toString()));

41
application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java

@ -25,6 +25,7 @@ import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.web.context.request.async.DeferredResult; import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfile;
@ -33,6 +34,7 @@ import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.ApiUsageStateId;
import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceId;
@ -53,8 +55,10 @@ import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.AccessControlService;
@ -111,6 +115,9 @@ public class AccessValidator {
@Autowired @Autowired
protected AccessControlService accessControlService; protected AccessControlService accessControlService;
@Autowired
protected ApiUsageStateService apiUsageStateService;
private ExecutorService executor; private ExecutorService executor;
@PostConstruct @PostConstruct
@ -152,7 +159,11 @@ public class AccessValidator {
new FutureCallback<DeferredResult<ResponseEntity>>() { new FutureCallback<DeferredResult<ResponseEntity>>() {
@Override @Override
public void onSuccess(@Nullable DeferredResult<ResponseEntity> result) { public void onSuccess(@Nullable DeferredResult<ResponseEntity> result) {
onSuccess.accept(response, currentUser.getTenantId(), entityId); try {
onSuccess.accept(response, currentUser.getTenantId(), entityId);
} catch (Exception e) {
onFailure(e);
}
} }
@Override @Override
@ -193,6 +204,9 @@ public class AccessValidator {
case ENTITY_VIEW: case ENTITY_VIEW:
validateEntityView(currentUser, operation, entityId, callback); validateEntityView(currentUser, operation, entityId, callback);
return; return;
case API_USAGE_STATE:
validateApiUsageState(currentUser, operation, entityId, callback);
return;
default: default:
//TODO: add support of other entities //TODO: add support of other entities
throw new IllegalStateException("Not Implemented!"); throw new IllegalStateException("Not Implemented!");
@ -237,6 +251,27 @@ public class AccessValidator {
} }
} }
private void validateApiUsageState(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback<ValidationResult> callback) {
if (currentUser.isSystemAdmin()) {
callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION));
} else {
if (!operation.equals(Operation.READ_TELEMETRY)) {
callback.onSuccess(ValidationResult.accessDenied("Allowed only READ_TELEMETRY operation!"));
}
ApiUsageState apiUsageState = apiUsageStateService.findApiUsageStateById(currentUser.getTenantId(), new ApiUsageStateId(entityId.getId()));
if (apiUsageState == null) {
callback.onSuccess(ValidationResult.entityNotFound("Api Usage State with requested id wasn't found!"));
} else {
try {
accessControlService.checkPermission(currentUser, Resource.API_USAGE_STATE, operation, entityId, apiUsageState);
} catch (ThingsboardException e) {
callback.onSuccess(ValidationResult.accessDenied(e.getMessage()));
}
callback.onSuccess(ValidationResult.ok(apiUsageState));
}
}
}
private void validateAsset(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback<ValidationResult> callback) { private void validateAsset(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback<ValidationResult> callback) {
if (currentUser.isSystemAdmin()) { if (currentUser.isSystemAdmin()) {
callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION)); callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION));
@ -404,9 +439,9 @@ public class AccessValidator {
public static void handleError(Throwable e, final DeferredResult<ResponseEntity> response, HttpStatus defaultErrorStatus) { public static void handleError(Throwable e, final DeferredResult<ResponseEntity> response, HttpStatus defaultErrorStatus) {
ResponseEntity responseEntity; ResponseEntity responseEntity;
if (e != null && e instanceof ToErrorResponseEntity) { if (e instanceof ToErrorResponseEntity) {
responseEntity = ((ToErrorResponseEntity) e).toErrorResponseEntity(); responseEntity = ((ToErrorResponseEntity) e).toErrorResponseEntity();
} else if (e != null && e instanceof IllegalArgumentException) { } else if (e instanceof IllegalArgumentException || e instanceof IncorrectParameterException) {
responseEntity = new ResponseEntity<>(e.getMessage(), HttpStatus.BAD_REQUEST); responseEntity = new ResponseEntity<>(e.getMessage(), HttpStatus.BAD_REQUEST);
} else { } else {
responseEntity = new ResponseEntity<>(defaultErrorStatus); responseEntity = new ResponseEntity<>(defaultErrorStatus);

2
application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationFailureHandler.java

@ -30,7 +30,7 @@ import java.nio.charset.StandardCharsets;
@Component(value = "oauth2AuthenticationFailureHandler") @Component(value = "oauth2AuthenticationFailureHandler")
@ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true") @ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true")
public class Oauth2AuthenticationFailureHandler extends SimpleUrlAuthenticationFailureHandler { public class Oauth2AuthenticationFailureHandler extends SimpleUrlAuthenticationFailureHandler {
@Override @Override
public void onAuthenticationFailure(HttpServletRequest request, public void onAuthenticationFailure(HttpServletRequest request,

1
application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java

@ -63,7 +63,6 @@ public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationS
public void onAuthenticationSuccess(HttpServletRequest request, public void onAuthenticationSuccess(HttpServletRequest request,
HttpServletResponse response, HttpServletResponse response,
Authentication authentication) throws IOException { Authentication authentication) throws IOException {
String baseUrl = MiscUtils.constructBaseUrl(request); String baseUrl = MiscUtils.constructBaseUrl(request);
try { try {
OAuth2AuthenticationToken token = (OAuth2AuthenticationToken) authentication; OAuth2AuthenticationToken token = (OAuth2AuthenticationToken) authentication;

3
application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java

@ -35,7 +35,8 @@ public enum Resource {
OAUTH2_CONFIGURATION_INFO(), OAUTH2_CONFIGURATION_INFO(),
OAUTH2_CONFIGURATION_TEMPLATE(), OAUTH2_CONFIGURATION_TEMPLATE(),
TENANT_PROFILE(EntityType.TENANT_PROFILE), TENANT_PROFILE(EntityType.TENANT_PROFILE),
DEVICE_PROFILE(EntityType.DEVICE_PROFILE); DEVICE_PROFILE(EntityType.DEVICE_PROFILE),
API_USAGE_STATE(EntityType.API_USAGE_STATE);
private final EntityType entityType; private final EntityType entityType;

1
application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java

@ -40,6 +40,7 @@ public class TenantAdminPermissions extends AbstractPermissions {
put(Resource.WIDGETS_BUNDLE, widgetsPermissionChecker); put(Resource.WIDGETS_BUNDLE, widgetsPermissionChecker);
put(Resource.WIDGET_TYPE, widgetsPermissionChecker); put(Resource.WIDGET_TYPE, widgetsPermissionChecker);
put(Resource.DEVICE_PROFILE, tenantEntityPermissionChecker); put(Resource.DEVICE_PROFILE, tenantEntityPermissionChecker);
put(Resource.API_USAGE_STATE, tenantEntityPermissionChecker);
} }
public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() {

24
application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java

@ -40,17 +40,20 @@ import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.security.model.SecuritySettings;
import org.thingsboard.server.common.data.security.model.UserPasswordPolicy;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.dao.user.UserServiceImpl; import org.thingsboard.server.dao.user.UserServiceImpl;
import org.thingsboard.server.service.security.exception.UserPasswordExpiredException; import org.thingsboard.server.service.security.exception.UserPasswordExpiredException;
import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.utils.MiscUtils;
import org.thingsboard.server.common.data.security.model.UserPasswordPolicy;
import javax.annotation.Resource; import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -146,7 +149,7 @@ public class DefaultSystemSecurityService implements SystemSecurityService {
if (isPositiveInteger(securitySettings.getPasswordPolicy().getPasswordExpirationPeriodDays())) { if (isPositiveInteger(securitySettings.getPasswordPolicy().getPasswordExpirationPeriodDays())) {
if ((userCredentials.getCreatedTime() if ((userCredentials.getCreatedTime()
+ TimeUnit.DAYS.toMillis(securitySettings.getPasswordPolicy().getPasswordExpirationPeriodDays())) + TimeUnit.DAYS.toMillis(securitySettings.getPasswordPolicy().getPasswordExpirationPeriodDays()))
< System.currentTimeMillis()) { < System.currentTimeMillis()) {
userCredentials = userService.requestExpiredPasswordReset(tenantId, userCredentials.getId()); userCredentials = userService.requestExpiredPasswordReset(tenantId, userCredentials.getId());
throw new UserPasswordExpiredException("User password expired!", userCredentials.getResetToken()); throw new UserPasswordExpiredException("User password expired!", userCredentials.getResetToken());
} }
@ -197,6 +200,21 @@ public class DefaultSystemSecurityService implements SystemSecurityService {
} }
} }
@Override
public String getBaseUrl(TenantId tenantId, CustomerId customerId, HttpServletRequest httpServletRequest) {
String baseUrl;
AdminSettings generalSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "general");
JsonNode prohibitDifferentUrl = generalSettings.getJsonValue().get("prohibitDifferentUrl");
if (prohibitDifferentUrl != null && prohibitDifferentUrl.asBoolean()) {
baseUrl = generalSettings.getJsonValue().get("baseUrl").asText();
} else {
baseUrl = MiscUtils.constructBaseUrl(httpServletRequest);
}
return baseUrl;
}
private static boolean isPositiveInteger(Integer val) { private static boolean isPositiveInteger(Integer val) {
return val != null && val.intValue() > 0; return val != null && val.intValue() > 0;
} }

5
application/src/main/java/org/thingsboard/server/service/security/system/SystemSecurityService.java

@ -16,11 +16,14 @@
package org.thingsboard.server.service.security.system; package org.thingsboard.server.service.security.system;
import org.springframework.security.core.AuthenticationException; import org.springframework.security.core.AuthenticationException;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.security.model.SecuritySettings;
import javax.servlet.http.HttpServletRequest;
public interface SystemSecurityService { public interface SystemSecurityService {
SecuritySettings getSecuritySettings(TenantId tenantId); SecuritySettings getSecuritySettings(TenantId tenantId);
@ -31,4 +34,6 @@ public interface SystemSecurityService {
void validatePassword(TenantId tenantId, String password, UserCredentials userCredentials) throws DataValidationException; void validatePassword(TenantId tenantId, String password, UserCredentials userCredentials) throws DataValidationException;
String getBaseUrl(TenantId tenantId, CustomerId customerId, HttpServletRequest httpServletRequest);
} }

8
application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java

@ -48,9 +48,9 @@ import java.util.stream.Collectors;
public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService { public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService {
public static final String TB_SERVICE_QUEUE = "TbServiceQueue"; public static final String TB_SERVICE_QUEUE = "TbServiceQueue";
public static final FutureCallback<Void> CALLBACK = new FutureCallback<Void>() { public static final FutureCallback<Integer> CALLBACK = new FutureCallback<Integer>() {
@Override @Override
public void onSuccess(@Nullable Void result) { public void onSuccess(@Nullable Integer result) {
} }
@ -85,7 +85,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS
.map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get())))
.collect(Collectors.toList()); .collect(Collectors.toList());
if (!tsList.isEmpty()) { if (!tsList.isEmpty()) {
tsService.saveAndNotify(tenantId, serviceAssetId, tsList, CALLBACK); tsService.saveAndNotifyInternal(tenantId, serviceAssetId, tsList, CALLBACK);
} }
} }
} catch (DataValidationException e) { } catch (DataValidationException e) {
@ -97,7 +97,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS
ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> {
TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString())); TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString()));
try { try {
tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); tsService.saveAndNotifyInternal(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK);
} catch (DataValidationException e2) { } catch (DataValidationException e2) {
if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) {
throw e2; throw e2;

6
application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java

@ -105,10 +105,10 @@ public abstract class AbstractSubscriptionService implements ApplicationListener
} }
} }
protected void addWsCallback(ListenableFuture<List<Void>> saveFuture, Consumer<Void> callback) { protected <T> void addWsCallback(ListenableFuture<T> saveFuture, Consumer<T> callback) {
Futures.addCallback(saveFuture, new FutureCallback<List<Void>>() { Futures.addCallback(saveFuture, new FutureCallback<T>() {
@Override @Override
public void onSuccess(@Nullable List<Void> result) { public void onSuccess(@Nullable T result) {
callback.accept(null); callback.accept(null);
} }

109
application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java

@ -20,10 +20,9 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityView;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
@ -41,11 +40,12 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.queue.TbClusterService;
import org.thingsboard.server.service.subscription.SubscriptionManagerService;
import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import org.thingsboard.server.service.subscription.TbSubscriptionUtils;
import javax.annotation.Nullable; import javax.annotation.Nullable;
@ -59,12 +59,8 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.function.Consumer;
import java.util.stream.Collectors;
/** /**
* Created by ashvayka on 27.03.18. * Created by ashvayka on 27.03.18.
@ -76,6 +72,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
private final AttributesService attrService; private final AttributesService attrService;
private final TimeseriesService tsService; private final TimeseriesService tsService;
private final EntityViewService entityViewService; private final EntityViewService entityViewService;
private final TbApiUsageClient apiUsageClient;
private final TbApiUsageStateService apiUsageStateService;
private ExecutorService tsCallBackExecutor; private ExecutorService tsCallBackExecutor;
@ -83,11 +81,15 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
TimeseriesService tsService, TimeseriesService tsService,
EntityViewService entityViewService, EntityViewService entityViewService,
TbClusterService clusterService, TbClusterService clusterService,
PartitionService partitionService) { PartitionService partitionService,
TbApiUsageClient apiUsageClient,
TbApiUsageStateService apiUsageStateService) {
super(clusterService, partitionService); super(clusterService, partitionService);
this.attrService = attrService; this.attrService = attrService;
this.tsService = tsService; this.tsService = tsService;
this.entityViewService = entityViewService; this.entityViewService = entityViewService;
this.apiUsageClient = apiUsageClient;
this.apiUsageStateService = apiUsageStateService;
} }
@PostConstruct @PostConstruct
@ -116,7 +118,35 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
@Override @Override
public void saveAndNotify(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, long ttl, FutureCallback<Void> callback) { public void saveAndNotify(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, long ttl, FutureCallback<Void> callback) {
ListenableFuture<List<Void>> saveFuture = tsService.save(tenantId, entityId, ts, ttl); checkInternalEntity(entityId);
if (apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) {
saveAndNotifyInternal(tenantId, entityId, ts, ttl, new FutureCallback<Integer>() {
@Override
public void onSuccess(Integer result) {
if (result != null && result > 0) {
apiUsageClient.report(tenantId, ApiUsageRecordKey.STORAGE_DP_COUNT, result);
}
callback.onSuccess(null);
}
@Override
public void onFailure(Throwable t) {
callback.onFailure(t);
}
});
} else{
callback.onFailure(new RuntimeException("DB storage writes are disabled due to API limits!"));
}
}
@Override
public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Integer> callback) {
saveAndNotifyInternal(tenantId, entityId, ts, 0L, callback);
}
@Override
public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, long ttl, FutureCallback<Integer> callback) {
ListenableFuture<Integer> saveFuture = tsService.save(tenantId, entityId, ts, ttl);
addMainCallback(saveFuture, callback); addMainCallback(saveFuture, callback);
addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts));
if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) {
@ -141,9 +171,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
Optional<TsKvEntry> tsKvEntry = entries.stream() Optional<TsKvEntry> tsKvEntry = entries.stream()
.filter(entry -> entry.getTs() > startTs && entry.getTs() <= endTs) .filter(entry -> entry.getTs() > startTs && entry.getTs() <= endTs)
.max(Comparator.comparingLong(TsKvEntry::getTs)); .max(Comparator.comparingLong(TsKvEntry::getTs));
if (tsKvEntry.isPresent()) { tsKvEntry.ifPresent(entityViewLatest::add);
entityViewLatest.add(tsKvEntry.get());
}
} }
} }
if (!entityViewLatest.isEmpty()) { if (!entityViewLatest.isEmpty()) {
@ -176,29 +204,53 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
@Override @Override
public void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, boolean notifyDevice, FutureCallback<Void> callback) { public void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, boolean notifyDevice, FutureCallback<Void> callback) {
checkInternalEntity(entityId);
saveAndNotifyInternal(tenantId, entityId, scope, attributes, notifyDevice, callback);
}
@Override
public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, boolean notifyDevice, FutureCallback<Void> callback) {
ListenableFuture<List<Void>> saveFuture = attrService.save(tenantId, entityId, scope, attributes); ListenableFuture<List<Void>> saveFuture = attrService.save(tenantId, entityId, scope, attributes);
addMainCallback(saveFuture, callback); addVoidCallback(saveFuture, callback);
addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice));
} }
@Override @Override
public void saveLatestAndNotify(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Void> callback) { public void saveLatestAndNotify(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Void> callback) {
checkInternalEntity(entityId);
saveLatestAndNotifyInternal(tenantId, entityId, ts, callback);
}
@Override
public void saveLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Void> callback) {
ListenableFuture<List<Void>> saveFuture = tsService.saveLatest(tenantId, entityId, ts); ListenableFuture<List<Void>> saveFuture = tsService.saveLatest(tenantId, entityId, ts);
addMainCallback(saveFuture, callback); addVoidCallback(saveFuture, callback);
addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts));
} }
@Override @Override
public void deleteAndNotify(TenantId tenantId, EntityId entityId, String scope, List<String> keys, FutureCallback<Void> callback) { public void deleteAndNotify(TenantId tenantId, EntityId entityId, String scope, List<String> keys, FutureCallback<Void> callback) {
checkInternalEntity(entityId);
deleteAndNotifyInternal(tenantId, entityId, scope, keys, callback);
}
@Override
public void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List<String> keys, FutureCallback<Void> callback) {
ListenableFuture<List<Void>> deleteFuture = attrService.removeAll(tenantId, entityId, scope, keys); ListenableFuture<List<Void>> deleteFuture = attrService.removeAll(tenantId, entityId, scope, keys);
addMainCallback(deleteFuture, callback); addVoidCallback(deleteFuture, callback);
addWsCallback(deleteFuture, success -> onAttributesDelete(tenantId, entityId, scope, keys)); addWsCallback(deleteFuture, success -> onAttributesDelete(tenantId, entityId, scope, keys));
} }
@Override @Override
public void deleteLatest(TenantId tenantId, EntityId entityId, List<String> keys, FutureCallback<Void> callback) { public void deleteLatest(TenantId tenantId, EntityId entityId, List<String> keys, FutureCallback<Void> callback) {
checkInternalEntity(entityId);
deleteLatestInternal(tenantId, entityId, keys, callback);
}
@Override
public void deleteLatestInternal(TenantId tenantId, EntityId entityId, List<String> keys, FutureCallback<Void> callback) {
ListenableFuture<List<Void>> deleteFuture = tsService.removeLatest(tenantId, entityId, keys); ListenableFuture<List<Void>> deleteFuture = tsService.removeLatest(tenantId, entityId, keys);
addMainCallback(deleteFuture, callback); addVoidCallback(deleteFuture, callback);
} }
@Override @Override
@ -283,7 +335,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
} }
} }
private <S, R> void addMainCallback(ListenableFuture<S> saveFuture, final FutureCallback<R> callback) { private <S> void addVoidCallback(ListenableFuture<S> saveFuture, final FutureCallback<Void> callback) {
Futures.addCallback(saveFuture, new FutureCallback<S>() { Futures.addCallback(saveFuture, new FutureCallback<S>() {
@Override @Override
public void onSuccess(@Nullable S result) { public void onSuccess(@Nullable S result) {
@ -296,4 +348,25 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
} }
}, tsCallBackExecutor); }, tsCallBackExecutor);
} }
private <S> void addMainCallback(ListenableFuture<S> saveFuture, final FutureCallback<S> callback) {
Futures.addCallback(saveFuture, new FutureCallback<S>() {
@Override
public void onSuccess(@Nullable S result) {
callback.onSuccess(result);
}
@Override
public void onFailure(Throwable t) {
callback.onFailure(t);
}
}, tsCallBackExecutor);
}
private void checkInternalEntity(EntityId entityId) {
if (EntityType.API_USAGE_STATE.equals(entityId.getEntityType())) {
throw new RuntimeException("Can't update API Usage State!");
}
}
} }

46
application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java

@ -0,0 +1,46 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.telemetry;
import com.google.common.util.concurrent.FutureCallback;
import org.thingsboard.rule.engine.api.RuleEngineTelemetryService;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import java.util.List;
/**
* Created by ashvayka on 27.03.18.
*/
public interface InternalTelemetryService extends RuleEngineTelemetryService {
void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Integer> callback);
void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, long ttl, FutureCallback<Integer> callback);
void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, boolean notifyDevice, FutureCallback<Void> callback);
void saveLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Void> callback);
void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List<String> keys, FutureCallback<Void> callback);
void deleteLatestInternal(TenantId tenantId, EntityId entityId, List<String> keys, FutureCallback<Void> callback);
}

2
application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java

@ -22,6 +22,6 @@ import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
/** /**
* Created by ashvayka on 27.03.18. * Created by ashvayka on 27.03.18.
*/ */
public interface TelemetrySubscriptionService extends RuleEngineTelemetryService, ApplicationListener<PartitionChangeEvent> { public interface TelemetrySubscriptionService extends InternalTelemetryService, ApplicationListener<PartitionChangeEvent> {
} }

47
application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java

@ -25,6 +25,7 @@ import com.google.protobuf.ByteString;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils; import org.springframework.util.StringUtils;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfile;
@ -51,7 +52,6 @@ import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.device.provision.ProvisionRequest; import org.thingsboard.server.dao.device.provision.ProvisionRequest;
import org.thingsboard.server.dao.device.provision.ProvisionResponse; import org.thingsboard.server.dao.device.provision.ProvisionResponse;
import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.util.mapping.JacksonUtil; import org.thingsboard.server.dao.util.mapping.JacksonUtil;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto;
@ -68,9 +68,10 @@ import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509Ce
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.dao.device.provision.ProvisionFailedException; import org.thingsboard.server.dao.device.provision.ProvisionFailedException;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.service.profile.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.queue.TbClusterService;
import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.state.DeviceStateService;
@ -92,6 +93,7 @@ public class DefaultTransportApiService implements TransportApiService {
private final TbDeviceProfileCache deviceProfileCache; private final TbDeviceProfileCache deviceProfileCache;
private final TbTenantProfileCache tenantProfileCache; private final TbTenantProfileCache tenantProfileCache;
private final TbApiUsageStateService apiUsageStateService;
private final DeviceService deviceService; private final DeviceService deviceService;
private final RelationService relationService; private final RelationService relationService;
private final DeviceCredentialsService deviceCredentialsService; private final DeviceCredentialsService deviceCredentialsService;
@ -104,13 +106,14 @@ public class DefaultTransportApiService implements TransportApiService {
private final ConcurrentMap<String, ReentrantLock> deviceCreationLocks = new ConcurrentHashMap<>(); private final ConcurrentMap<String, ReentrantLock> deviceCreationLocks = new ConcurrentHashMap<>();
public DefaultTransportApiService(TbDeviceProfileCache deviceProfileCache, public DefaultTransportApiService(TbDeviceProfileCache deviceProfileCache,
TbTenantProfileCache tenantProfileCache, DeviceService deviceService, TbTenantProfileCache tenantProfileCache, TbApiUsageStateService apiUsageStateService, DeviceService deviceService,
RelationService relationService, DeviceCredentialsService deviceCredentialsService, RelationService relationService, DeviceCredentialsService deviceCredentialsService,
DeviceStateService deviceStateService, DbCallbackExecutorService dbCallbackExecutorService, DeviceStateService deviceStateService, DbCallbackExecutorService dbCallbackExecutorService,
TbClusterService tbClusterService, DataDecodingEncodingService dataDecodingEncodingService, TbClusterService tbClusterService, DataDecodingEncodingService dataDecodingEncodingService,
DeviceProvisionService deviceProvisionService) { DeviceProvisionService deviceProvisionService) {
this.deviceProfileCache = deviceProfileCache; this.deviceProfileCache = deviceProfileCache;
this.tenantProfileCache = tenantProfileCache; this.tenantProfileCache = tenantProfileCache;
this.apiUsageStateService = apiUsageStateService;
this.deviceService = deviceService; this.deviceService = deviceService;
this.relationService = relationService; this.relationService = relationService;
this.deviceCredentialsService = deviceCredentialsService; this.deviceCredentialsService = deviceCredentialsService;
@ -161,22 +164,25 @@ public class DefaultTransportApiService implements TransportApiService {
} }
private ListenableFuture<TransportApiResponseMsg> validateCredentials(TransportProtos.ValidateBasicMqttCredRequestMsg mqtt) { private ListenableFuture<TransportApiResponseMsg> validateCredentials(TransportProtos.ValidateBasicMqttCredRequestMsg mqtt) {
DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByCredentialsId(mqtt.getUserName()); DeviceCredentials credentials = null;
if (credentials != null) { if (mqtt.getUserName() != null) {
if (credentials.getCredentialsType() == DeviceCredentialsType.ACCESS_TOKEN) { credentials = deviceCredentialsService.findDeviceCredentialsByCredentialsId(mqtt.getUserName());
return getDeviceInfo(credentials.getDeviceId(), credentials); if (credentials != null) {
} else if (credentials.getCredentialsType() == DeviceCredentialsType.MQTT_BASIC) { if (credentials.getCredentialsType() == DeviceCredentialsType.ACCESS_TOKEN) {
if (!checkMqttCredentials(mqtt, credentials)) { return getDeviceInfo(credentials.getDeviceId(), credentials);
credentials = null; } else if (credentials.getCredentialsType() == DeviceCredentialsType.MQTT_BASIC) {
if (!checkMqttCredentials(mqtt, credentials)) {
credentials = null;
}
} }
} }
}
if (credentials == null) {
credentials = checkMqttCredentials(mqtt, EncryptionUtil.getSha3Hash("|", mqtt.getClientId(), mqtt.getUserName()));
if (credentials == null) { if (credentials == null) {
credentials = checkMqttCredentials(mqtt, EncryptionUtil.getSha3Hash(mqtt.getClientId())); credentials = checkMqttCredentials(mqtt, EncryptionUtil.getSha3Hash("|", mqtt.getClientId(), mqtt.getUserName()));
} }
} }
if (credentials == null) {
credentials = checkMqttCredentials(mqtt, EncryptionUtil.getSha3Hash(mqtt.getClientId()));
}
if (credentials != null) { if (credentials != null) {
return getDeviceInfo(credentials.getDeviceId(), credentials); return getDeviceInfo(credentials.getDeviceId(), credentials);
} else { } else {
@ -316,18 +322,21 @@ public class DefaultTransportApiService implements TransportApiService {
private ListenableFuture<TransportApiResponseMsg> handle(GetEntityProfileRequestMsg requestMsg) { private ListenableFuture<TransportApiResponseMsg> handle(GetEntityProfileRequestMsg requestMsg) {
EntityType entityType = EntityType.valueOf(requestMsg.getEntityType()); EntityType entityType = EntityType.valueOf(requestMsg.getEntityType());
UUID entityUuid = new UUID(requestMsg.getEntityIdMSB(), requestMsg.getEntityIdLSB()); UUID entityUuid = new UUID(requestMsg.getEntityIdMSB(), requestMsg.getEntityIdLSB());
ByteString data; GetEntityProfileResponseMsg.Builder builder = GetEntityProfileResponseMsg.newBuilder();
if (entityType.equals(EntityType.DEVICE_PROFILE)) { if (entityType.equals(EntityType.DEVICE_PROFILE)) {
DeviceProfileId deviceProfileId = new DeviceProfileId(entityUuid); DeviceProfileId deviceProfileId = new DeviceProfileId(entityUuid);
DeviceProfile deviceProfile = deviceProfileCache.find(deviceProfileId); DeviceProfile deviceProfile = deviceProfileCache.find(deviceProfileId);
data = ByteString.copyFrom(dataDecodingEncodingService.encode(deviceProfile)); builder.setData(ByteString.copyFrom(dataDecodingEncodingService.encode(deviceProfile)));
} else if (entityType.equals(EntityType.TENANT)) { } else if (entityType.equals(EntityType.TENANT)) {
TenantProfile tenantProfile = tenantProfileCache.get(new TenantId(entityUuid)); TenantId tenantId = new TenantId(entityUuid);
data = ByteString.copyFrom(dataDecodingEncodingService.encode(tenantProfile)); TenantProfile tenantProfile = tenantProfileCache.get(tenantId);
ApiUsageState state = apiUsageStateService.getApiUsageState(tenantId);
builder.setData(ByteString.copyFrom(dataDecodingEncodingService.encode(tenantProfile)));
builder.setApiState(ByteString.copyFrom(dataDecodingEncodingService.encode(state)));
} else { } else {
throw new RuntimeException("Invalid entity profile request: " + entityType); throw new RuntimeException("Invalid entity profile request: " + entityType);
} }
return Futures.immediateFuture(TransportApiResponseMsg.newBuilder().setEntityProfileResponseMsg(GetEntityProfileResponseMsg.newBuilder().setData(data).build()).build()); return Futures.immediateFuture(TransportApiResponseMsg.newBuilder().setEntityProfileResponseMsg(builder).build());
} }
private ListenableFuture<TransportApiResponseMsg> getDeviceInfo(DeviceId deviceId, DeviceCredentials credentials) { private ListenableFuture<TransportApiResponseMsg> getDeviceInfo(DeviceId deviceId, DeviceCredentials credentials) {

2
application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java

@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener; import org.springframework.context.event.EventListener;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.server.common.stats.MessagesStats; import org.thingsboard.server.common.stats.MessagesStats;
import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsFactory;
@ -90,6 +91,7 @@ public class TbCoreTransportApiService {
} }
@EventListener(ApplicationReadyEvent.class) @EventListener(ApplicationReadyEvent.class)
@Order(value = 2)
public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
log.info("Received application ready event. Starting polling for events."); log.info("Received application ready event. Starting polling for events.");
transportApiTemplate.init(transportApiService); transportApiTemplate.init(transportApiService);

2
application/src/main/resources/thingsboard.yml

@ -673,6 +673,7 @@ queue:
poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}"
partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}"
pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}"
usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}"
stats: stats:
enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}"
print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}"
@ -765,7 +766,6 @@ metrics:
# Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,).
percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" percentiles: "${METRICS_TIMER_PERCENTILES:0.5}"
management: management:
endpoints: endpoints:
web: web:

7
application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java

@ -23,7 +23,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityInfo;
import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
@ -285,7 +286,9 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController
TenantProfile tenantProfile = new TenantProfile(); TenantProfile tenantProfile = new TenantProfile();
tenantProfile.setName(name); tenantProfile.setName(name);
tenantProfile.setDescription(name + " Test"); tenantProfile.setDescription(name + " Test");
tenantProfile.setProfileData(new TenantProfileData()); TenantProfileData tenantProfileData = new TenantProfileData();
tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration());
tenantProfile.setProfileData(tenantProfileData);
tenantProfile.setDefault(false); tenantProfile.setDefault(false);
tenantProfile.setIsolatedTbCore(false); tenantProfile.setIsolatedTbCore(false);
tenantProfile.setIsolatedTbRuleEngine(false); tenantProfile.setIsolatedTbRuleEngine(false);

4
application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java

@ -27,8 +27,8 @@ import java.util.Arrays;
@RunWith(ClasspathSuite.class) @RunWith(ClasspathSuite.class)
@ClasspathSuite.ClassnameFilters({ @ClasspathSuite.ClassnameFilters({
// "org.thingsboard.server.controller.sql.WebsocketApiSqlTest", // "org.thingsboard.server.controller.sql.WebsocketApiSqlTest",
// "org.thingsboard.server.controller.sql.EntityQueryControllerSqlTest", // "org.thingsboard.server.controller.sql.TenantProfileControllerSqlTest",
"org.thingsboard.server.controller.sql.DeviceProfileControllerSqlTest", "org.thingsboard.server.controller.sql.*Test",
}) })
public class ControllerSqlTestSuite { public class ControllerSqlTestSuite {

250
application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java

@ -1,250 +0,0 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.script;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import com.google.common.collect.Sets;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.rule.engine.api.ScriptEngine;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgDataType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import javax.script.ScriptException;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.Assert.*;
public class RuleNodeJsScriptEngineTest {
private ScriptEngine scriptEngine;
private TestNashornJsInvokeService jsSandboxService;
private EntityId ruleNodeId = new RuleNodeId(Uuids.timeBased());
@Before
public void beforeTest() throws Exception {
jsSandboxService = new TestNashornJsInvokeService(false, 1, 100, 3);
}
@After
public void afterTest() throws Exception {
jsSandboxService.stop();
}
@Test
public void msgCanBeUpdated() throws ScriptException {
String function = "metadata.temp = metadata.temp * 10; return {metadata: metadata};";
scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function);
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("temp", "7");
metaData.putValue("humidity", "99");
String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}";
TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson);
TbMsg actual = scriptEngine.executeUpdate(msg);
assertEquals("70", actual.getMetaData().getValue("temp"));
scriptEngine.destroy();
}
@Test
public void newAttributesCanBeAddedInMsg() throws ScriptException {
String function = "metadata.newAttr = metadata.humidity - msg.passed; return {metadata: metadata};";
scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function);
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("temp", "7");
metaData.putValue("humidity", "99");
String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}";
TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson);
TbMsg actual = scriptEngine.executeUpdate(msg);
assertEquals("94", actual.getMetaData().getValue("newAttr"));
scriptEngine.destroy();
}
@Test
public void payloadCanBeUpdated() throws ScriptException {
String function = "msg.passed = msg.passed * metadata.temp; msg.bigObj.newProp = 'Ukraine'; return {msg: msg};";
scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function);
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("temp", "7");
metaData.putValue("humidity", "99");
String rawJson = "{\"name\":\"Vit\",\"passed\": 5,\"bigObj\":{\"prop\":42}}";
TbMsg msg =TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson);
TbMsg actual = scriptEngine.executeUpdate(msg);
String expectedJson = "{\"name\":\"Vit\",\"passed\":35,\"bigObj\":{\"prop\":42,\"newProp\":\"Ukraine\"}}";
assertEquals(expectedJson, actual.getData());
scriptEngine.destroy();
}
@Test
public void metadataAccessibleForFilter() throws ScriptException {
String function = "return metadata.humidity < 15;";
scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function);
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("temp", "7");
metaData.putValue("humidity", "99");
String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}";
TbMsg msg = TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson);
assertFalse(scriptEngine.executeFilter(msg));
scriptEngine.destroy();
}
@Test
public void dataAccessibleForFilter() throws ScriptException {
String function = "return msg.passed < 15 && msg.name === 'Vit' && metadata.temp == 7 && msg.bigObj.prop == 42;";
scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function);
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("temp", "7");
metaData.putValue("humidity", "99");
String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}";
TbMsg msg = TbMsg.newMsg( "USER", null, metaData,TbMsgDataType.JSON, rawJson);
assertTrue(scriptEngine.executeFilter(msg));
scriptEngine.destroy();
}
@Test
public void dataAccessibleForSwitch() throws ScriptException {
String jsCode = "function nextRelation(metadata, msg) {\n" +
" if(msg.passed == 5 && metadata.temp == 10)\n" +
" return 'one'\n" +
" else\n" +
" return 'two';\n" +
"};\n" +
"\n" +
"return nextRelation(metadata, msg);";
scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, jsCode);
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("temp", "10");
metaData.putValue("humidity", "99");
String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}";
TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson);
Set<String> actual = scriptEngine.executeSwitch(msg);
assertEquals(Sets.newHashSet("one"), actual);
scriptEngine.destroy();
}
@Test
public void multipleRelationsReturnedFromSwitch() throws ScriptException {
String jsCode = "function nextRelation(metadata, msg) {\n" +
" if(msg.passed == 5 && metadata.temp == 10)\n" +
" return ['three', 'one']\n" +
" else\n" +
" return 'two';\n" +
"};\n" +
"\n" +
"return nextRelation(metadata, msg);";
scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, jsCode);
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("temp", "10");
metaData.putValue("humidity", "99");
String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}";
TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson);
Set<String> actual = scriptEngine.executeSwitch(msg);
assertEquals(Sets.newHashSet("one", "three"), actual);
scriptEngine.destroy();
}
@Test
public void concurrentReleasedCorrectly() throws InterruptedException, ExecutionException {
String code = "metadata.temp = metadata.temp * 10; return {metadata: metadata};";
int repeat = 1000;
ExecutorService service = Executors.newFixedThreadPool(repeat);
Map<UUID, Object> scriptIds = new ConcurrentHashMap<>();
CountDownLatch startLatch = new CountDownLatch(repeat);
CountDownLatch finishLatch = new CountDownLatch(repeat);
AtomicInteger failedCount = new AtomicInteger(0);
for (int i = 0; i < repeat; i++) {
service.submit(() -> runScript(startLatch, finishLatch, failedCount, scriptIds, code));
}
finishLatch.await();
assertTrue(scriptIds.size() == 1);
assertTrue(failedCount.get() == 0);
CountDownLatch nextStart = new CountDownLatch(repeat);
CountDownLatch nextFinish = new CountDownLatch(repeat);
for (int i = 0; i < repeat; i++) {
service.submit(() -> runScript(nextStart, nextFinish, failedCount, scriptIds, code));
}
nextFinish.await();
assertTrue(scriptIds.size() == 1);
assertTrue(failedCount.get() == 0);
service.shutdownNow();
}
@Test
public void concurrentFailedEvaluationShouldThrowException() throws InterruptedException {
String code = "metadata.temp = metadata.temp * 10; urn {metadata: metadata};";
int repeat = 10000;
ExecutorService service = Executors.newFixedThreadPool(repeat);
Map<UUID, Object> scriptIds = new ConcurrentHashMap<>();
CountDownLatch startLatch = new CountDownLatch(repeat);
CountDownLatch finishLatch = new CountDownLatch(repeat);
AtomicInteger failedCount = new AtomicInteger(0);
for (int i = 0; i < repeat; i++) {
service.submit(() -> {
service.submit(() -> runScript(startLatch, finishLatch, failedCount, scriptIds, code));
});
}
finishLatch.await();
assertTrue(scriptIds.isEmpty());
assertEquals(repeat, failedCount.get());
service.shutdownNow();
}
private void runScript(CountDownLatch startLatch, CountDownLatch finishLatch, AtomicInteger failedCount,
Map<UUID, Object> scriptIds, String code) {
try {
for (int k = 0; k < 10; k++) {
startLatch.countDown();
startLatch.await();
UUID scriptId = jsSandboxService.eval(JsScriptType.RULE_NODE_SCRIPT, code).get();
scriptIds.put(scriptId, new Object());
jsSandboxService.invokeFunction(scriptId, "{}", "{}", "TEXT").get();
jsSandboxService.release(scriptId).get();
}
} catch (Throwable th) {
failedCount.incrementAndGet();
} finally {
finishLatch.countDown();
}
}
}

57
application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java

@ -1,57 +0,0 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.script;
public class TestNashornJsInvokeService extends AbstractNashornJsInvokeService {
private boolean useJsSandbox;
private final int monitorThreadPoolSize;
private final long maxCpuTime;
private final int maxErrors;
public TestNashornJsInvokeService(boolean useJsSandbox, int monitorThreadPoolSize, long maxCpuTime, int maxErrors) {
this.useJsSandbox = useJsSandbox;
this.monitorThreadPoolSize = monitorThreadPoolSize;
this.maxCpuTime = maxCpuTime;
this.maxErrors = maxErrors;
init();
}
@Override
protected boolean useJsSandbox() {
return useJsSandbox;
}
@Override
protected int getMonitorThreadPoolSize() {
return monitorThreadPoolSize;
}
@Override
protected long getMaxCpuTime() {
return maxCpuTime;
}
@Override
protected int getMaxErrors() {
return maxErrors;
}
@Override
protected long getMaxBlacklistDuration() {
return 100000;
}
}

2
common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateService.java

@ -31,4 +31,6 @@ public interface RuleNodeStateService {
RuleNodeState save(TenantId tenantId, RuleNodeState ruleNodeState); RuleNodeState save(TenantId tenantId, RuleNodeState ruleNodeState);
void removeByRuleNodeId(TenantId tenantId, RuleNodeId selfId); void removeByRuleNodeId(TenantId tenantId, RuleNodeId selfId);
void removeByRuleNodeIdAndEntityId(TenantId tenantId, RuleNodeId selfId, EntityId entityId);
} }

2
application/src/main/java/org/thingsboard/server/service/profile/TbTenantProfileCache.java → common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TbTenantProfileCache.java

@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.thingsboard.server.service.profile; package org.thingsboard.server.dao.tenant;
import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;

4
common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java

@ -36,9 +36,9 @@ public interface TimeseriesService {
ListenableFuture<List<TsKvEntry>> findAllLatest(TenantId tenantId, EntityId entityId); ListenableFuture<List<TsKvEntry>> findAllLatest(TenantId tenantId, EntityId entityId);
ListenableFuture<List<Void>> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); ListenableFuture<Integer> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry);
ListenableFuture<List<Void>> save(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntry, long ttl); ListenableFuture<Integer> save(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntry, long ttl);
ListenableFuture<List<Void>> saveLatest(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntry); ListenableFuture<List<Void>> saveLatest(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntry);

33
common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java

@ -0,0 +1,33 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.usagerecord;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.id.ApiUsageStateId;
import org.thingsboard.server.common.data.id.TenantId;
public interface ApiUsageStateService {
ApiUsageState createDefaultApiUsageState(TenantId id);
ApiUsageState update(ApiUsageState apiUsageState);
ApiUsageState findTenantApiUsageState(TenantId tenantId);
void deleteApiUsageStateByTenantId(TenantId tenantId);
ApiUsageState findApiUsageStateById(TenantId tenantId, ApiUsageStateId id);
}

29
common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileData.java → common/data/src/main/java/org/thingsboard/server/common/data/ApiFeature.java

@ -15,28 +15,19 @@
*/ */
package org.thingsboard.server.common.data; package org.thingsboard.server.common.data;
import com.fasterxml.jackson.annotation.JsonAnyGetter; import lombok.Getter;
import com.fasterxml.jackson.annotation.JsonAnySetter;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Data;
import java.util.HashMap; public enum ApiFeature {
import java.util.Map; TRANSPORT("transportApiState"),
DB("dbApiState"),
RE("ruleEngineApiState"),
JS("jsExecutionApiState");
@Data @Getter
public class TenantProfileData { private final String apiStateKey;
@JsonIgnore ApiFeature(String apiStateKey) {
private Map<String, Object> properties = new HashMap<>(); this.apiStateKey = apiStateKey;
@JsonAnyGetter
public Map<String, Object> properties() {
return this.properties;
}
@JsonAnySetter
public void put(String name, Object value) {
this.properties.put(name, value);
} }
} }

60
common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java

@ -0,0 +1,60 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data;
import lombok.Getter;
public enum ApiUsageRecordKey {
TRANSPORT_MSG_COUNT(ApiFeature.TRANSPORT, "transportMsgCount", "transportMsgLimit"),
TRANSPORT_DP_COUNT(ApiFeature.TRANSPORT, "transportDataPointsCount", "transportDataPointsLimit"),
STORAGE_DP_COUNT(ApiFeature.DB, "storageDataPointsCount", "storageDataPointsLimit"),
RE_EXEC_COUNT(ApiFeature.RE, "ruleEngineExecutionCount", "ruleEngineExecutionLimit"),
JS_EXEC_COUNT(ApiFeature.JS, "jsExecutionCount", "jsExecutionLimit");
private static final ApiUsageRecordKey[] JS_RECORD_KEYS = {JS_EXEC_COUNT};
private static final ApiUsageRecordKey[] RE_RECORD_KEYS = {RE_EXEC_COUNT};
private static final ApiUsageRecordKey[] DB_RECORD_KEYS = {STORAGE_DP_COUNT};
private static final ApiUsageRecordKey[] TRANSPORT_RECORD_KEYS = {TRANSPORT_MSG_COUNT, TRANSPORT_DP_COUNT};
@Getter
private final ApiFeature apiFeature;
@Getter
private final String apiCountKey;
@Getter
private final String apiLimitKey;
ApiUsageRecordKey(ApiFeature apiFeature, String apiCountKey, String apiLimitKey) {
this.apiFeature = apiFeature;
this.apiCountKey = apiCountKey;
this.apiLimitKey = apiLimitKey;
}
public static ApiUsageRecordKey[] getKeys(ApiFeature feature) {
switch (feature) {
case TRANSPORT:
return TRANSPORT_RECORD_KEYS;
case DB:
return DB_RECORD_KEYS;
case RE:
return RE_RECORD_KEYS;
case JS:
return JS_RECORD_KEYS;
default:
return new ApiUsageRecordKey[]{};
}
}
}

84
common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java

@ -0,0 +1,84 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.ApiUsageStateId;
@ToString
@EqualsAndHashCode(callSuper = true)
public class ApiUsageState extends BaseData<ApiUsageStateId> implements HasTenantId {
private static final long serialVersionUID = 8250339805336035966L;
@Getter
@Setter
private TenantId tenantId;
@Getter
@Setter
private EntityId entityId;
@Getter
@Setter
private ApiUsageStateValue transportState;
@Getter
@Setter
private ApiUsageStateValue dbStorageState;
@Getter
@Setter
private ApiUsageStateValue reExecState;
@Getter
@Setter
private ApiUsageStateValue jsExecState;
public ApiUsageState() {
super();
}
public ApiUsageState(ApiUsageStateId id) {
super(id);
}
public ApiUsageState(ApiUsageState ur) {
super(ur);
this.tenantId = ur.getTenantId();
this.entityId = ur.getEntityId();
this.transportState = ur.getTransportState();
this.dbStorageState = ur.getDbStorageState();
this.reExecState = ur.getReExecState();
this.jsExecState = ur.getJsExecState();
}
public boolean isTransportEnabled() {
return !ApiUsageStateValue.DISABLED.equals(transportState);
}
public boolean isReExecEnabled() {
return !ApiUsageStateValue.DISABLED.equals(reExecState);
}
public boolean isDbStorageEnabled() {
return !ApiUsageStateValue.DISABLED.equals(dbStorageState);
}
public boolean isJsExecEnabled() {
return !ApiUsageStateValue.DISABLED.equals(jsExecState);
}
}

26
common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageStateValue.java

@ -0,0 +1,26 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data;
public enum ApiUsageStateValue {
ENABLED, WARNING, DISABLED;
public static ApiUsageStateValue toMoreRestricted(ApiUsageStateValue a, ApiUsageStateValue b) {
return a.ordinal() > b.ordinal() ? a : b;
}
}

2
common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java

@ -43,6 +43,7 @@ public class DeviceProfile extends SearchTextBased<DeviceProfileId> implements H
private DeviceTransportType transportType; private DeviceTransportType transportType;
private DeviceProfileProvisionType provisionType; private DeviceProfileProvisionType provisionType;
private RuleChainId defaultRuleChainId; private RuleChainId defaultRuleChainId;
private String defaultQueueName;
private transient DeviceProfileData profileData; private transient DeviceProfileData profileData;
@JsonIgnore @JsonIgnore
private byte[] profileDataBytes; private byte[] profileDataBytes;
@ -63,6 +64,7 @@ public class DeviceProfile extends SearchTextBased<DeviceProfileId> implements H
this.description = deviceProfile.getDescription(); this.description = deviceProfile.getDescription();
this.isDefault = deviceProfile.isDefault(); this.isDefault = deviceProfile.isDefault();
this.defaultRuleChainId = deviceProfile.getDefaultRuleChainId(); this.defaultRuleChainId = deviceProfile.getDefaultRuleChainId();
this.defaultQueueName = deviceProfile.getDefaultQueueName();
this.setProfileData(deviceProfile.getProfileData()); this.setProfileData(deviceProfile.getProfileData());
this.provisionDeviceKey = deviceProfile.getProvisionDeviceKey(); this.provisionDeviceKey = deviceProfile.getProvisionDeviceKey();
} }

2
common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java

@ -19,5 +19,5 @@ package org.thingsboard.server.common.data;
* @author Andrew Shvayka * @author Andrew Shvayka
*/ */
public enum EntityType { public enum EntityType {
TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, RULE_CHAIN, RULE_NODE, ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, TENANT_PROFILE, DEVICE_PROFILE TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, RULE_CHAIN, RULE_NODE, ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, TENANT_PROFILE, DEVICE_PROFILE, API_USAGE_STATE;
} }

12
common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java

@ -21,6 +21,8 @@ import lombok.Data;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.id.TenantProfileId;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
@ -78,15 +80,21 @@ public class TenantProfile extends SearchTextBased<TenantProfileId> implements H
profileData = mapper.readValue(new ByteArrayInputStream(profileDataBytes), TenantProfileData.class); profileData = mapper.readValue(new ByteArrayInputStream(profileDataBytes), TenantProfileData.class);
} catch (IOException e) { } catch (IOException e) {
log.warn("Can't deserialize tenant profile data: ", e); log.warn("Can't deserialize tenant profile data: ", e);
return null; return createDefaultTenantProfileData();
} }
return profileData; return profileData;
} else { } else {
return null; return createDefaultTenantProfileData();
} }
} }
} }
public TenantProfileData createDefaultTenantProfileData() {
TenantProfileData tpd = new TenantProfileData();
tpd.setConfiguration(new DefaultTenantProfileConfiguration());
return tpd;
}
public void setProfileData(TenantProfileData data) { public void setProfileData(TenantProfileData data) {
this.profileData = data; this.profileData = data;
try { try {

20
common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileType.java

@ -0,0 +1,20 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data;
public enum TenantProfileType {
DEFAULT
}

3
common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java

@ -18,6 +18,7 @@ package org.thingsboard.server.common.data.device.profile;
import lombok.Data; import lombok.Data;
import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmSeverity;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -27,7 +28,7 @@ public class DeviceProfileAlarm {
private String id; private String id;
private String alarmType; private String alarmType;
private Map<AlarmSeverity, AlarmRule> createRules; private LinkedHashMap<AlarmSeverity, AlarmRule> createRules;
private AlarmRule clearRule; private AlarmRule clearRule;
// Hidden in advanced settings // Hidden in advanced settings

42
common/data/src/main/java/org/thingsboard/server/common/data/id/ApiUsageStateId.java

@ -0,0 +1,42 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.id;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.thingsboard.server.common.data.EntityType;
import java.util.UUID;
public class ApiUsageStateId extends UUIDBased implements EntityId {
@JsonCreator
public ApiUsageStateId(@JsonProperty("id") UUID id) {
super(id);
}
public static ApiUsageStateId fromString(String userId) {
return new ApiUsageStateId(UUID.fromString(userId));
}
@JsonIgnore
@Override
public EntityType getEntityType() {
return EntityType.API_USAGE_STATE;
}
}

2
common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java

@ -66,6 +66,8 @@ public class EntityIdFactory {
return new DeviceProfileId(uuid); return new DeviceProfileId(uuid);
case TENANT_PROFILE: case TENANT_PROFILE:
return new TenantProfileId(uuid); return new TenantProfileId(uuid);
case API_USAGE_STATE:
return new ApiUsageStateId(uuid);
} }
throw new IllegalArgumentException("EntityType " + type + " is not supported!"); throw new IllegalArgumentException("EntityType " + type + " is not supported!");
} }

19
common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java

@ -19,7 +19,7 @@ import java.util.Objects;
import java.util.Optional; import java.util.Optional;
public class BasicTsKvEntry implements TsKvEntry { public class BasicTsKvEntry implements TsKvEntry {
private static final int MAX_CHARS_PER_DATA_POINT = 512;
private final long ts; private final long ts;
private final KvEntry kv; private final KvEntry kv;
@ -99,4 +99,21 @@ public class BasicTsKvEntry implements TsKvEntry {
public String getValueAsString() { public String getValueAsString() {
return kv.getValueAsString(); return kv.getValueAsString();
} }
@Override
public int getDataPoints() {
int length;
switch (getDataType()) {
case STRING:
length = getStrValue().get().length();
break;
case JSON:
length = getJsonValue().get().length();
break;
default:
return 1;
}
return Math.max(1, (length + MAX_CHARS_PER_DATA_POINT - 1) / MAX_CHARS_PER_DATA_POINT);
}
} }

5
common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java

@ -15,6 +15,8 @@
*/ */
package org.thingsboard.server.common.data.kv; package org.thingsboard.server.common.data.kv;
import com.fasterxml.jackson.annotation.JsonIgnore;
/** /**
* Represents time series KV data entry * Represents time series KV data entry
* *
@ -25,4 +27,7 @@ public interface TsKvEntry extends KvEntry {
long getTs(); long getTs();
@JsonIgnore
int getDataPoints();
} }

2
common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java

@ -21,5 +21,5 @@ import java.io.Serializable;
* @author Andrew Shvayka * @author Andrew Shvayka
*/ */
public enum ComponentLifecycleEvent implements Serializable { public enum ComponentLifecycleEvent implements Serializable {
CREATED, STARTED, ACTIVATED, SUSPENDED, UPDATED, STOPPED, DELETED CREATED, STARTED, ACTIVATED, SUSPENDED, UPDATED, STOPPED, DELETED, ADDED_TO_ALLOW_LIST, ADDED_TO_DENY_LIST
} }

28
common/data/src/main/java/org/thingsboard/server/common/data/query/ApiUsageStateFilter.java

@ -0,0 +1,28 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.query;
import lombok.Data;
import org.thingsboard.server.common.data.id.EntityId;
@Data
public class ApiUsageStateFilter implements EntityFilter {
@Override
public EntityFilterType getType() {
return EntityFilterType.API_USAGE_STATE;
}
}

1
common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java

@ -32,6 +32,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
@JsonSubTypes.Type(value = AssetTypeFilter.class, name = "assetType"), @JsonSubTypes.Type(value = AssetTypeFilter.class, name = "assetType"),
@JsonSubTypes.Type(value = DeviceTypeFilter.class, name = "deviceType"), @JsonSubTypes.Type(value = DeviceTypeFilter.class, name = "deviceType"),
@JsonSubTypes.Type(value = EntityViewTypeFilter.class, name = "entityViewType"), @JsonSubTypes.Type(value = EntityViewTypeFilter.class, name = "entityViewType"),
@JsonSubTypes.Type(value = ApiUsageStateFilter.class, name = "apiUsageState"),
@JsonSubTypes.Type(value = RelationsQueryFilter.class, name = "relationsQuery"), @JsonSubTypes.Type(value = RelationsQueryFilter.class, name = "relationsQuery"),
@JsonSubTypes.Type(value = AssetSearchQueryFilter.class, name = "assetSearchQuery"), @JsonSubTypes.Type(value = AssetSearchQueryFilter.class, name = "assetSearchQuery"),
@JsonSubTypes.Type(value = DeviceSearchQueryFilter.class, name = "deviceSearchQuery"), @JsonSubTypes.Type(value = DeviceSearchQueryFilter.class, name = "deviceSearchQuery"),

3
common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilterType.java

@ -25,7 +25,8 @@ public enum EntityFilterType {
RELATIONS_QUERY("relationsQuery"), RELATIONS_QUERY("relationsQuery"),
ASSET_SEARCH_QUERY("assetSearchQuery"), ASSET_SEARCH_QUERY("assetSearchQuery"),
DEVICE_SEARCH_QUERY("deviceSearchQuery"), DEVICE_SEARCH_QUERY("deviceSearchQuery"),
ENTITY_VIEW_SEARCH_QUERY("entityViewSearchQuery"); ENTITY_VIEW_SEARCH_QUERY("entityViewSearchQuery"),
API_USAGE_STATE("apiUsageState");
private final String label; private final String label;

75
common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java

@ -0,0 +1,75 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.tenant.profile;
import lombok.Data;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.TenantProfileType;
@Data
public class DefaultTenantProfileConfiguration implements TenantProfileConfiguration {
private long maxDevices;
private long maxAssets;
private String transportTenantMsgRateLimit;
private String transportTenantTelemetryMsgRateLimit;
private String transportTenantTelemetryDataPointsRateLimit;
private String transportDeviceMsgRateLimit;
private String transportDeviceTelemetryMsgRateLimit;
private String transportDeviceTelemetryDataPointsRateLimit;
private long maxTransportMessages;
private long maxTransportDataPoints;
private long maxREExecutions;
private long maxJSExecutions;
private long maxDPStorageDays;
private int maxRuleNodeExecutionsPerMessage;
private double warnThreshold;
@Override
public long getProfileThreshold(ApiUsageRecordKey key) {
switch (key) {
case TRANSPORT_MSG_COUNT:
return maxTransportMessages;
case TRANSPORT_DP_COUNT:
return maxTransportDataPoints;
case JS_EXEC_COUNT:
return maxJSExecutions;
case RE_EXEC_COUNT:
return maxREExecutions;
case STORAGE_DP_COUNT:
return maxDPStorageDays;
}
return 0L;
}
@Override
public long getWarnThreshold(ApiUsageRecordKey key) {
return (long) (getProfileThreshold(key) * (warnThreshold > 0.0 ? warnThreshold : 0.8));
}
@Override
public TenantProfileType getType() {
return TenantProfileType.DEFAULT;
}
@Override
public int getMaxRuleNodeExecsPerMessage() {
return maxRuleNodeExecutionsPerMessage;
}
}

46
common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileConfiguration.java

@ -0,0 +1,46 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.tenant.profile;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.TenantProfileType;
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
include = JsonTypeInfo.As.PROPERTY,
property = "type")
@JsonSubTypes({
@JsonSubTypes.Type(value = DefaultTenantProfileConfiguration.class, name = "DEFAULT")})
public interface TenantProfileConfiguration {
@JsonIgnore
TenantProfileType getType();
@JsonIgnore
long getProfileThreshold(ApiUsageRecordKey key);
@JsonIgnore
long getWarnThreshold(ApiUsageRecordKey key);
@JsonIgnore
int getMaxRuleNodeExecsPerMessage();
}

9
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitFactory.java → common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java

@ -13,12 +13,13 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.thingsboard.server.common.transport.limits; package org.thingsboard.server.common.data.tenant.profile;
public interface TransportRateLimitFactory { import lombok.Data;
TransportRateLimit create(TransportRateLimitType type, Object config); @Data
public class TenantProfileData {
TransportRateLimit createDefault(TransportRateLimitType type); private TenantProfileConfiguration configuration;
} }

64
common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java

@ -18,27 +18,27 @@ package org.thingsboard.server.common.msg;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.protobuf.ByteString; import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.InvalidProtocolBufferException;
import lombok.AccessLevel;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.gen.MsgProtos;
import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
import org.thingsboard.server.common.msg.queue.ServiceQueue; import org.thingsboard.server.common.msg.queue.ServiceQueue;
import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import java.io.IOException;
import java.io.Serializable; import java.io.Serializable;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
/** /**
* Created by ashvayka on 13.01.18. * Created by ashvayka on 13.01.18.
*/ */
@Data @Data
@Builder
@Slf4j @Slf4j
public final class TbMsg implements Serializable { public final class TbMsg implements Serializable {
@ -52,51 +52,73 @@ public final class TbMsg implements Serializable {
private final String data; private final String data;
private final RuleChainId ruleChainId; private final RuleChainId ruleChainId;
private final RuleNodeId ruleNodeId; private final RuleNodeId ruleNodeId;
@Getter(value = AccessLevel.NONE)
private final AtomicInteger ruleNodeExecCounter;
public int getAndIncrementRuleNodeCounter() {
return ruleNodeExecCounter.getAndIncrement();
}
//This field is not serialized because we use queues and there is no need to do it //This field is not serialized because we use queues and there is no need to do it
@JsonIgnore @JsonIgnore
transient private final TbMsgCallback callback; transient private final TbMsgCallback callback;
public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator,
metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, 0, TbMsgCallback.EMPTY);
} }
public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) {
return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, TbMsgCallback.EMPTY);
} }
// REALLY NEW MSG
public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data) { public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data) {
return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, TbMsgCallback.EMPTY);
} }
public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data) { public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data) {
return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, null, null, TbMsgCallback.EMPTY); return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, null, null, 0, TbMsgCallback.EMPTY);
} }
// For Tests only
public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, 0, TbMsgCallback.EMPTY);
} }
public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, TbMsgCallback callback) { public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, TbMsgCallback callback) {
return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, callback); return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, callback);
}
public static TbMsg transformMsg(TbMsg tbMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) {
return new TbMsg(tbMsg.getQueueName(), tbMsg.getId(), tbMsg.getTs(), type, originator, metaData.copy(), tbMsg.getDataType(),
data, tbMsg.getRuleChainId(), tbMsg.getRuleNodeId(), tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback());
}
public static TbMsg transformMsg(TbMsg tbMsg, RuleChainId ruleChainId) {
return new TbMsg(tbMsg.queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType,
tbMsg.data, ruleChainId, null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback());
} }
public static TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { public static TbMsg transformMsg(TbMsg tbMsg, String queueName) {
return new TbMsg(origMsg.getQueueName(), origMsg.getId(), origMsg.getTs(), type, originator, metaData.copy(), origMsg.getDataType(), return new TbMsg(queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType,
data, origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); tbMsg.data, tbMsg.getRuleChainId(), null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback());
} }
public static TbMsg transformMsg(TbMsg origMsg, RuleChainId ruleChainId) { public static TbMsg transformMsg(TbMsg tbMsg, RuleChainId ruleChainId, String queueName) {
return new TbMsg(origMsg.queueName, origMsg.id, origMsg.ts, origMsg.type, origMsg.originator, origMsg.metaData, origMsg.dataType, return new TbMsg(queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType,
origMsg.data, ruleChainId, null, origMsg.getCallback()); tbMsg.data, ruleChainId, null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback());
} }
public static TbMsg newMsg(TbMsg tbMsg, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { public static TbMsg newMsg(TbMsg tbMsg, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
return new TbMsg(tbMsg.getQueueName(), UUID.randomUUID(), tbMsg.getTs(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(), return new TbMsg(tbMsg.getQueueName(), UUID.randomUUID(), tbMsg.getTs(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(),
tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, tbMsg.ruleNodeExecCounter.get(), TbMsgCallback.EMPTY);
} }
private TbMsg(String queueName, UUID id, long ts, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, private TbMsg(String queueName, UUID id, long ts, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data,
RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { RuleChainId ruleChainId, RuleNodeId ruleNodeId, int ruleNodeExecCounter, TbMsgCallback callback) {
this.id = id; this.id = id;
this.queueName = queueName; this.queueName = queueName;
if (ts > 0) { if (ts > 0) {
@ -111,6 +133,7 @@ public final class TbMsg implements Serializable {
this.data = data; this.data = data;
this.ruleChainId = ruleChainId; this.ruleChainId = ruleChainId;
this.ruleNodeId = ruleNodeId; this.ruleNodeId = ruleNodeId;
this.ruleNodeExecCounter = new AtomicInteger(ruleNodeExecCounter);
if (callback != null) { if (callback != null) {
this.callback = callback; this.callback = callback;
} else { } else {
@ -147,6 +170,7 @@ public final class TbMsg implements Serializable {
builder.setDataType(msg.getDataType().ordinal()); builder.setDataType(msg.getDataType().ordinal());
builder.setData(msg.getData()); builder.setData(msg.getData());
builder.setRuleNodeExecCounter(msg.ruleNodeExecCounter.get());
return builder.build().toByteArray(); return builder.build().toByteArray();
} }
@ -164,18 +188,18 @@ public final class TbMsg implements Serializable {
ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB())); ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB()));
} }
TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()];
return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, callback); return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, proto.getRuleNodeExecCounter(), callback);
} catch (InvalidProtocolBufferException e) { } catch (InvalidProtocolBufferException e) {
throw new IllegalStateException("Could not parse protobuf for TbMsg", e); throw new IllegalStateException("Could not parse protobuf for TbMsg", e);
} }
} }
public TbMsg copyWithRuleChainId(RuleChainId ruleChainId) { public TbMsg copyWithRuleChainId(RuleChainId ruleChainId) {
return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, callback); return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, this.ruleNodeExecCounter.get(), callback);
} }
public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) { public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, callback); return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, this.ruleNodeExecCounter.get(), callback);
} }
public TbMsgCallback getCallback() { public TbMsgCallback getCallback() {

4
common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java

@ -50,6 +50,10 @@ public class TopicPartitionInfo {
this.fullTopicName = tmp; this.fullTopicName = tmp;
} }
public TopicPartitionInfo newByTopic(String topic) {
return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.myPartition);
}
public String getTopic() { public String getTopic() {
return topic; return topic;
} }

48
common/message/src/main/java/org/thingsboard/server/common/msg/tools/SchedulerUtils.java

@ -15,16 +15,64 @@
*/ */
package org.thingsboard.server.common.msg.tools; package org.thingsboard.server.common.msg.tools;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZoneId; import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.temporal.ChronoUnit;
import java.time.temporal.TemporalAdjuster;
import java.time.temporal.TemporalAdjusters;
import java.time.temporal.TemporalUnit;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import static java.time.temporal.ChronoField.DAY_OF_MONTH;
import static java.time.temporal.ChronoUnit.MONTHS;
public class SchedulerUtils { public class SchedulerUtils {
private final static ZoneId UTC = ZoneId.of("UTC");
private static final ConcurrentMap<String, ZoneId> tzMap = new ConcurrentHashMap<>(); private static final ConcurrentMap<String, ZoneId> tzMap = new ConcurrentHashMap<>();
public static ZoneId getZoneId(String tz) { public static ZoneId getZoneId(String tz) {
return tzMap.computeIfAbsent(tz == null || tz.isEmpty() ? "UTC" : tz, ZoneId::of); return tzMap.computeIfAbsent(tz == null || tz.isEmpty() ? "UTC" : tz, ZoneId::of);
} }
public static long getStartOfCurrentHour() {
return getStartOfCurrentHour(UTC);
}
public static long getStartOfCurrentHour(ZoneId zoneId) {
return LocalDateTime.now(ZoneOffset.UTC).atZone(zoneId).truncatedTo(ChronoUnit.HOURS).toInstant().toEpochMilli();
}
public static long getStartOfCurrentMonth() {
return getStartOfCurrentMonth(UTC);
}
public static long getStartOfCurrentMonth(ZoneId zoneId) {
return LocalDate.now().withDayOfMonth(1).atStartOfDay(zoneId).toInstant().toEpochMilli();
}
public static long getStartOfNextMonth() {
return getStartOfNextMonth(UTC);
}
public static long getStartOfNextMonth(ZoneId zoneId) {
return LocalDate.now().with(TemporalAdjusters.firstDayOfNextMonth()).atStartOfDay(zoneId).toInstant().toEpochMilli();
}
public static long getStartOfNextNextMonth() {
return getStartOfNextNextMonth(UTC);
}
public static long getStartOfNextNextMonth(ZoneId zoneId) {
return LocalDate.now().with(firstDayOfNextNextMonth()).atStartOfDay(zoneId).toInstant().toEpochMilli();
}
public static TemporalAdjuster firstDayOfNextNextMonth() {
return (temporal) -> temporal.with(DAY_OF_MONTH, 1).plus(2, MONTHS);
}
} }

1
common/message/src/main/proto/tbmsg.proto

@ -45,4 +45,5 @@ message TbMsgProto {
string data = 14; string data = 14;
int64 ts = 15; int64 ts = 15;
int32 ruleNodeExecCounter = 16;
} }

45
common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java

@ -0,0 +1,45 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.common;
import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.queue.TbQueueCallback;
import org.thingsboard.server.queue.TbQueueMsgMetadata;
import java.util.concurrent.atomic.AtomicInteger;
public class MultipleTbQueueCallbackWrapper implements TbQueueCallback {
private final AtomicInteger tbQueueCallbackCount;
private final TbQueueCallback callback;
public MultipleTbQueueCallbackWrapper(int tbQueueCallbackCount, TbQueueCallback callback) {
this.tbQueueCallbackCount = new AtomicInteger(tbQueueCallbackCount);
this.callback = callback;
}
@Override
public void onSuccess(TbQueueMsgMetadata metadata) {
if (tbQueueCallbackCount.decrementAndGet() <= 0) {
callback.onSuccess(metadata);
}
}
@Override
public void onFailure(Throwable t) {
callback.onFailure(new RuleEngineException(t.getMessage()));
}
}

2
common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java

@ -20,6 +20,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.annotation.DependsOn; import org.springframework.context.annotation.DependsOn;
import org.springframework.context.event.EventListener; import org.springframework.context.event.EventListener;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.util.Collections; import java.util.Collections;
@ -40,6 +41,7 @@ public class DummyDiscoveryService implements DiscoveryService {
} }
@EventListener(ApplicationReadyEvent.class) @EventListener(ApplicationReadyEvent.class)
@Order(value = 1)
public void onApplicationEvent(ApplicationReadyEvent event) { public void onApplicationEvent(ApplicationReadyEvent event) {
partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), Collections.emptyList()); partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), Collections.emptyList());
} }

2
common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java

@ -34,6 +34,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener; import org.springframework.context.event.EventListener;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.Assert; import org.springframework.util.Assert;
import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.common.util.ThingsBoardThreadFactory;
@ -112,6 +113,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi
} }
@EventListener(ApplicationReadyEvent.class) @EventListener(ApplicationReadyEvent.class)
@Order(value = 1)
public void onApplicationEvent(ApplicationReadyEvent event) { public void onApplicationEvent(ApplicationReadyEvent event) {
if (stopped) { if (stopped) {
log.debug("Ignoring application ready event. Service is stopped."); log.debug("Ignoring application ready event. Service is stopped.");

45
common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java

@ -22,7 +22,7 @@ import org.springframework.stereotype.Component;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest;
import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.*;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
@ -91,64 +91,64 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToTransportMsg>> createTransportNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic()); return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> createRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> createTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) {
return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(), return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() {
return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings,
partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> createToCoreMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() {
return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(), return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() {
return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings,
partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>> createTransportApiRequestConsumer() { public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() {
return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(), return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> createTransportApiResponseProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() {
return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getResponsesTopic()); return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getResponsesTopic());
} }
@ -175,6 +175,17 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
return builder.build(); return builder.build();
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic());
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, coreSettings.getUsageStatsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

12
common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java

@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
@ -163,6 +164,17 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory {
return builder.build(); return builder.build();
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic());
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, coreSettings.getUsageStatsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

6
common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java

@ -143,6 +143,11 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory
return builder.build(); return builder.build();
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic());
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {
@ -158,4 +163,5 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory
notificationAdmin.destroy(); notificationAdmin.destroy();
} }
} }
} }

6
common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java

@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
@ -109,6 +110,11 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory {
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic());
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

12
common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java

@ -125,8 +125,20 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
return null; return null;
} }
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
return new InMemoryTbQueueConsumer<>(coreSettings.getUsageStatsTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new InMemoryTbQueueProducer<>(coreSettings.getUsageStatsTopic());
}
@Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}")
private void printInMemoryStats() { private void printInMemoryStats() {
storage.printStats(); storage.printStats();
} }
} }

7
common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java

@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
@ -96,4 +97,10 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory
public TbQueueConsumer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsConsumer() {
return new InMemoryTbQueueConsumer<>(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); return new InMemoryTbQueueConsumer<>(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId());
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new InMemoryTbQueueProducer<>(coreSettings.getUsageStatsTopic());
}
} }

23
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java

@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
@ -249,6 +250,28 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
return builder.build(); return builder.build();
} }
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> consumerBuilder = TbKafkaConsumerTemplate.builder();
consumerBuilder.settings(kafkaSettings);
consumerBuilder.topic(coreSettings.getUsageStatsTopic());
consumerBuilder.clientId("monolith-us-consumer-" + serviceInfoProvider.getServiceId());
consumerBuilder.groupId("monolith-us-consumer-" + serviceInfoProvider.getServiceId());
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(coreAdmin);
return consumerBuilder.build();
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> requestBuilder = TbKafkaProducerTemplate.builder();
requestBuilder.settings(kafkaSettings);
requestBuilder.clientId("monolith-us-producer-" + serviceInfoProvider.getServiceId());
requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic());
requestBuilder.admin(coreAdmin);
return requestBuilder.build();
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

23
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java

@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
@ -219,6 +220,28 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
return builder.build(); return builder.build();
} }
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> consumerBuilder = TbKafkaConsumerTemplate.builder();
consumerBuilder.settings(kafkaSettings);
consumerBuilder.topic(coreSettings.getUsageStatsTopic());
consumerBuilder.clientId("tb-core-us-consumer-" + serviceInfoProvider.getServiceId());
consumerBuilder.groupId("tb-core-us-consumer-" + serviceInfoProvider.getServiceId());
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(coreAdmin);
return consumerBuilder.build();
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> requestBuilder = TbKafkaProducerTemplate.builder();
requestBuilder.settings(kafkaSettings);
requestBuilder.clientId("tb-core-us-producer-" + serviceInfoProvider.getServiceId());
requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic());
requestBuilder.admin(coreAdmin);
return requestBuilder.build();
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

11
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java

@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
@ -192,6 +193,16 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
return builder.build(); return builder.build();
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> requestBuilder = TbKafkaProducerTemplate.builder();
requestBuilder.settings(kafkaSettings);
requestBuilder.clientId("tb-rule-engine-us-producer-" + serviceInfoProvider.getServiceId());
requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic());
requestBuilder.admin(coreAdmin);
return requestBuilder.build();
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

11
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java

@ -21,6 +21,7 @@ import org.springframework.stereotype.Component;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
@ -138,6 +139,16 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory {
return responseBuilder.build(); return responseBuilder.build();
} }
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> requestBuilder = TbKafkaProducerTemplate.builder();
requestBuilder.settings(kafkaSettings);
requestBuilder.clientId("transport-node-us-producer-" + serviceInfoProvider.getServiceId());
requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic());
requestBuilder.admin(coreAdmin);
return requestBuilder.build();
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

12
common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java

@ -27,6 +27,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
@ -183,6 +184,17 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
return builder.build(); return builder.build();
} }
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic());
}
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) { if (coreAdmin != null) {

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save