Browse Source

merge with master

pull/2685/head
YevhenBondarenko 7 years ago
parent
commit
6322da3d04
  1. 1
      application/src/main/conf/logback.xml
  2. 1274
      application/src/main/data/json/demo/dashboards/gateways.json
  3. 520
      application/src/main/data/json/demo/dashboards/rule_engine_statistics.json
  4. 1236
      application/src/main/data/json/demo/dashboards/theromstats.json
  5. 146
      application/src/main/data/json/demo/rule_chains/root_rule_chain.json
  6. 137
      application/src/main/data/json/demo/rule_chains/thermostat_alarms.json
  7. 32
      application/src/main/data/json/system/widget_bundles/gateway_widgets.json
  8. 25
      application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql
  9. 3
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  10. 3
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
  11. 5
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java
  12. 2
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java
  13. 40
      application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java
  14. 2
      application/src/main/java/org/thingsboard/server/controller/QueueController.java
  15. 14
      application/src/main/java/org/thingsboard/server/controller/RuleChainController.java
  16. 11
      application/src/main/java/org/thingsboard/server/controller/TelemetryController.java
  17. 2
      application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
  18. 34
      application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java
  19. 27
      application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java
  20. 5
      application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java
  21. 6
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java
  22. 15
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  23. 10
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
  24. 2
      application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java
  25. 8
      application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java
  26. 25
      application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java
  27. 28
      application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java
  28. 90
      application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java
  29. 11
      application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java
  30. 6
      application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java
  31. 16
      application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java
  32. 4
      application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java
  33. 10
      application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java
  34. 8
      application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java
  35. 2
      application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java
  36. 11
      application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java
  37. 8
      application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java
  38. 40
      application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java
  39. 59
      application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java
  40. 49
      application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java
  41. 2
      application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java
  42. 2
      application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java
  43. 3
      application/src/main/resources/logback.xml
  44. 107
      application/src/main/resources/thingsboard.yml
  45. 128
      application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java
  46. 6
      application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java
  47. 7
      application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java
  48. 7
      application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java
  49. 7
      application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java
  50. 7
      application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java
  51. 19
      application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java
  52. 11
      application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java
  53. 7
      application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java
  54. 7
      application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java
  55. 4
      application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java
  56. 21
      application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java
  57. 66
      application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java
  58. 7
      application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java
  59. 7
      application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java
  60. 38
      common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java
  61. 71
      common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java
  62. 12
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java
  63. 42
      common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java
  64. 11
      common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java
  65. 11
      common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java
  66. 9
      common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java
  67. 20
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java
  68. 19
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java
  69. 17
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java
  70. 18
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java
  71. 64
      common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java
  72. 54
      common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java
  73. 50
      common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java
  74. 54
      common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java
  75. 71
      common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java
  76. 57
      common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java
  77. 48
      common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java
  78. 50
      common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java
  79. 63
      common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java
  80. 57
      common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java
  81. 48
      common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java
  82. 51
      common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java
  83. 120
      common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java
  84. 3
      common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java
  85. 71
      common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java
  86. 11
      common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java
  87. 98
      common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java
  88. 2
      common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java
  89. 12
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java
  90. 8
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java
  91. 3
      dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java
  92. 14
      dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java
  93. 25
      dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java
  94. 3
      dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java
  95. 4
      dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java
  96. 2
      dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java
  97. 2
      dao/src/main/resources/sql/schema-entities-hsql.sql
  98. 26
      dao/src/main/resources/sql/schema-entities.sql
  99. 2
      dao/src/main/resources/sql/schema-timescale.sql
  100. 2
      dao/src/main/resources/sql/schema-ts-psql.sql

1
application/src/main/conf/logback.xml

@ -36,6 +36,7 @@
<logger name="org.thingsboard.server" level="INFO" /> <logger name="org.thingsboard.server" level="INFO" />
<logger name="akka" level="INFO" /> <logger name="akka" level="INFO" />
<logger name="com.microsoft.azure.servicebus.primitives.CoreMessageReceiver" level="OFF" />
<root level="INFO"> <root level="INFO">
<appender-ref ref="fileLogAppender"/> <appender-ref ref="fileLogAppender"/>

1274
application/src/main/data/json/demo/dashboards/gateways.json

File diff suppressed because it is too large

520
application/src/main/data/json/demo/dashboards/rule_engine_statistics.json

@ -0,0 +1,520 @@
{
"title": "Rule Engine Statistics",
"configuration": {
"widgets": {
"81987f19-3eac-e4ce-b790-d96e9b54d9a0": {
"isSystemType": true,
"bundleAlias": "charts",
"typeAlias": "basic_timeseries",
"type": "timeseries",
"title": "New widget",
"sizeX": 12,
"sizeY": 7,
"config": {
"datasources": [
{
"type": "entity",
"dataKeys": [
{
"name": "successfulMsgs",
"type": "timeseries",
"label": "${entityName} Successful",
"color": "#4caf50",
"settings": {
"excludeFromStacking": false,
"hideDataByDefault": false,
"disableDataHiding": false,
"removeFromLegend": false,
"showLines": true,
"fillLines": false,
"showPoints": false,
"showPointShape": "circle",
"pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);",
"showPointsLineWidth": 5,
"showPointsRadius": 3,
"showSeparateAxis": false,
"axisPosition": "left",
"thresholds": [
{
"thresholdValueSource": "predefinedValue"
}
],
"comparisonSettings": {
"showValuesForComparison": true
}
},
"_hash": 0.15490750967648736
},
{
"name": "failedMsgs",
"type": "timeseries",
"label": "${entityName} Permanent Failures",
"color": "#ef5350",
"settings": {
"excludeFromStacking": false,
"hideDataByDefault": false,
"disableDataHiding": false,
"removeFromLegend": false,
"showLines": true,
"fillLines": false,
"showPoints": false,
"showPointShape": "circle",
"pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);",
"showPointsLineWidth": 5,
"showPointsRadius": 3,
"showSeparateAxis": false,
"axisPosition": "left",
"thresholds": [
{
"thresholdValueSource": "predefinedValue"
}
],
"comparisonSettings": {
"showValuesForComparison": true
}
},
"_hash": 0.4186621166514697
},
{
"name": "tmpFailed",
"type": "timeseries",
"label": "${entityName} Processing Failures",
"color": "#ffc107",
"settings": {
"excludeFromStacking": false,
"hideDataByDefault": false,
"disableDataHiding": false,
"removeFromLegend": false,
"showLines": true,
"fillLines": false,
"showPoints": false,
"showPointShape": "circle",
"pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);",
"showPointsLineWidth": 5,
"showPointsRadius": 3,
"showSeparateAxis": false,
"axisPosition": "left",
"thresholds": [
{
"thresholdValueSource": "predefinedValue"
}
],
"comparisonSettings": {
"showValuesForComparison": true
}
},
"_hash": 0.49891007198715376
}
],
"entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018"
}
],
"timewindow": {
"realtime": {
"interval": 1000,
"timewindowMs": 300000
},
"aggregation": {
"type": "NONE",
"limit": 8640
},
"hideInterval": false,
"hideAggregation": false,
"hideAggInterval": false
},
"showTitle": true,
"backgroundColor": "#fff",
"color": "rgba(0, 0, 0, 0.87)",
"padding": "8px",
"settings": {
"shadowSize": 4,
"fontColor": "#545454",
"fontSize": 10,
"xaxis": {
"showLabels": true,
"color": "#545454"
},
"yaxis": {
"showLabels": true,
"color": "#545454"
},
"grid": {
"color": "#545454",
"tickColor": "#DDDDDD",
"verticalLines": true,
"horizontalLines": true,
"outlineWidth": 1
},
"stack": false,
"tooltipIndividual": false,
"timeForComparison": "months",
"xaxisSecond": {
"axisPosition": "top",
"showLabels": true
}
},
"title": "Queue Stats",
"dropShadow": true,
"enableFullscreen": true,
"titleStyle": {
"fontSize": "16px",
"fontWeight": 400
},
"mobileHeight": null,
"showTitleIcon": false,
"titleIcon": null,
"iconColor": "rgba(0, 0, 0, 0.87)",
"iconSize": "24px",
"titleTooltip": "",
"widgetStyle": {},
"useDashboardTimewindow": false,
"displayTimewindow": true,
"showLegend": true,
"actions": {},
"legendConfig": {
"direction": "column",
"position": "bottom",
"showMin": true,
"showMax": true,
"showAvg": false,
"showTotal": true
}
},
"id": "81987f19-3eac-e4ce-b790-d96e9b54d9a0"
},
"5eb79712-5c24-3060-7e4f-6af36b8f842d": {
"isSystemType": true,
"bundleAlias": "cards",
"typeAlias": "timeseries_table",
"type": "timeseries",
"title": "New widget",
"sizeX": 24,
"sizeY": 5,
"config": {
"datasources": [
{
"type": "entity",
"dataKeys": [
{
"name": "ruleEngineException",
"type": "timeseries",
"label": "Rule Chain",
"color": "#2196f3",
"settings": {
"useCellStyleFunction": false,
"useCellContentFunction": true,
"cellContentFunction": "return JSON.parse(value).ruleChainName;"
},
"_hash": 0.9954481282345906
},
{
"name": "ruleEngineException",
"type": "timeseries",
"label": "Rule Node",
"color": "#4caf50",
"settings": {
"useCellStyleFunction": false,
"useCellContentFunction": true,
"cellContentFunction": "return JSON.parse(value).ruleNodeName;"
},
"_hash": 0.18580357036589978
},
{
"name": "ruleEngineException",
"type": "timeseries",
"label": "Latest Error",
"color": "#f44336",
"settings": {
"useCellStyleFunction": false,
"useCellContentFunction": true,
"cellContentFunction": "return JSON.parse(value).message;"
},
"_hash": 0.7255162989552142
}
],
"entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018"
}
],
"timewindow": {
"realtime": {
"interval": 1000,
"timewindowMs": 86400000
},
"aggregation": {
"type": "NONE",
"limit": 200
}
},
"showTitle": true,
"backgroundColor": "rgb(255, 255, 255)",
"color": "rgba(0, 0, 0, 0.87)",
"padding": "8px",
"settings": {
"showTimestamp": true,
"displayPagination": true,
"defaultPageSize": 10
},
"title": "Exceptions",
"dropShadow": true,
"enableFullscreen": true,
"titleStyle": {
"fontSize": "16px",
"fontWeight": 400
},
"useDashboardTimewindow": false,
"showLegend": false,
"widgetStyle": {},
"actions": {},
"showTitleIcon": false,
"titleIcon": null,
"iconColor": "rgba(0, 0, 0, 0.87)",
"iconSize": "24px",
"titleTooltip": "",
"displayTimewindow": true
},
"id": "5eb79712-5c24-3060-7e4f-6af36b8f842d"
},
"ad3f1417-87a8-750e-fc67-49a2de1466d4": {
"isSystemType": true,
"bundleAlias": "charts",
"typeAlias": "basic_timeseries",
"type": "timeseries",
"title": "New widget",
"sizeX": 12,
"sizeY": 7,
"config": {
"datasources": [
{
"type": "entity",
"dataKeys": [
{
"name": "timeoutMsgs",
"type": "timeseries",
"label": "${entityName} Permanent Timeouts",
"color": "#4caf50",
"settings": {
"excludeFromStacking": false,
"hideDataByDefault": false,
"disableDataHiding": false,
"removeFromLegend": false,
"showLines": true,
"fillLines": false,
"showPoints": false,
"showPointShape": "circle",
"pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);",
"showPointsLineWidth": 5,
"showPointsRadius": 3,
"showSeparateAxis": false,
"axisPosition": "left",
"thresholds": [
{
"thresholdValueSource": "predefinedValue"
}
],
"comparisonSettings": {
"showValuesForComparison": true
}
},
"_hash": 0.565222981550328
},
{
"name": "tmpTimeout",
"type": "timeseries",
"label": "${entityName} Processing Timeouts",
"color": "#9c27b0",
"settings": {
"excludeFromStacking": false,
"hideDataByDefault": false,
"disableDataHiding": false,
"removeFromLegend": false,
"showLines": true,
"fillLines": false,
"showPoints": false,
"showPointShape": "circle",
"pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);",
"showPointsLineWidth": 5,
"showPointsRadius": 3,
"showSeparateAxis": false,
"axisPosition": "left",
"thresholds": [
{
"thresholdValueSource": "predefinedValue"
}
],
"comparisonSettings": {
"showValuesForComparison": true
}
},
"_hash": 0.2679547062508352
}
],
"entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018"
}
],
"timewindow": {
"realtime": {
"interval": 1000,
"timewindowMs": 300000
},
"aggregation": {
"type": "NONE",
"limit": 8640
},
"hideInterval": false,
"hideAggregation": false,
"hideAggInterval": false
},
"showTitle": true,
"backgroundColor": "#fff",
"color": "rgba(0, 0, 0, 0.87)",
"padding": "8px",
"settings": {
"shadowSize": 4,
"fontColor": "#545454",
"fontSize": 10,
"xaxis": {
"showLabels": true,
"color": "#545454"
},
"yaxis": {
"showLabels": true,
"color": "#545454"
},
"grid": {
"color": "#545454",
"tickColor": "#DDDDDD",
"verticalLines": true,
"horizontalLines": true,
"outlineWidth": 1
},
"stack": false,
"tooltipIndividual": false,
"timeForComparison": "months",
"xaxisSecond": {
"axisPosition": "top",
"showLabels": true
}
},
"title": "Processing Failures and Timeouts",
"dropShadow": true,
"enableFullscreen": true,
"titleStyle": {
"fontSize": "16px",
"fontWeight": 400
},
"mobileHeight": null,
"showTitleIcon": false,
"titleIcon": null,
"iconColor": "rgba(0, 0, 0, 0.87)",
"iconSize": "24px",
"titleTooltip": "",
"widgetStyle": {},
"useDashboardTimewindow": false,
"displayTimewindow": true,
"showLegend": true,
"actions": {},
"legendConfig": {
"direction": "column",
"position": "bottom",
"showMin": true,
"showMax": true,
"showAvg": false,
"showTotal": true
}
},
"id": "ad3f1417-87a8-750e-fc67-49a2de1466d4"
}
},
"states": {
"default": {
"name": "Rule Engine Statistics",
"root": true,
"layouts": {
"main": {
"widgets": {
"81987f19-3eac-e4ce-b790-d96e9b54d9a0": {
"sizeX": 12,
"sizeY": 7,
"mobileHeight": null,
"row": 0,
"col": 0
},
"5eb79712-5c24-3060-7e4f-6af36b8f842d": {
"sizeX": 24,
"sizeY": 5,
"row": 7,
"col": 0
},
"ad3f1417-87a8-750e-fc67-49a2de1466d4": {
"sizeX": 12,
"sizeY": 7,
"mobileHeight": null,
"row": 0,
"col": 12
}
},
"gridSettings": {
"backgroundColor": "#eeeeee",
"color": "rgba(0,0,0,0.870588)",
"columns": 24,
"margins": [
10,
10
],
"backgroundSizeMode": "100%",
"autoFillHeight": true,
"mobileAutoFillHeight": false,
"mobileRowHeight": 70
}
}
}
}
},
"entityAliases": {
"140f23dd-e3a0-ed98-6189-03c49d2d8018": {
"id": "140f23dd-e3a0-ed98-6189-03c49d2d8018",
"alias": "TbServiceQueues",
"filter": {
"type": "assetType",
"resolveMultiple": true,
"assetType": "TbServiceQueue",
"assetNameFilter": ""
}
}
},
"timewindow": {
"displayValue": "",
"selectedTab": 0,
"hideInterval": false,
"hideAggregation": false,
"hideAggInterval": false,
"realtime": {
"interval": 1000,
"timewindowMs": 60000
},
"history": {
"historyType": 0,
"interval": 1000,
"timewindowMs": 60000,
"fixedTimewindow": {
"startTimeMs": 1586176634823,
"endTimeMs": 1586263034823
}
},
"aggregation": {
"type": "AVG",
"limit": 25000
}
},
"settings": {
"stateControllerId": "entity",
"showTitle": false,
"showDashboardsSelect": true,
"showEntitiesSelect": true,
"showDashboardTimewindow": true,
"showDashboardExport": true,
"toolbarAlwaysOpen": true
}
},
"name": "Rule Engine Statistics"
}

1236
application/src/main/data/json/demo/dashboards/theromstats.json

File diff suppressed because it is too large

146
application/src/main/data/json/demo/rule_chains/root_rule_chain.json

@ -0,0 +1,146 @@
{
"ruleChain": {
"additionalInfo": null,
"name": "Root Rule Chain",
"firstRuleNodeId": null,
"root": true,
"debugMode": false,
"configuration": null
},
"metadata": {
"firstNodeIndex": 2,
"nodes": [
{
"additionalInfo": {
"layoutX": 824,
"layoutY": 156
},
"type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode",
"name": "Save Timeseries",
"debugMode": false,
"configuration": {
"defaultTTL": 0
}
},
{
"additionalInfo": {
"layoutX": 825,
"layoutY": 52
},
"type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode",
"name": "Save Client Attributes",
"debugMode": false,
"configuration": {
"scope": "CLIENT_SCOPE"
}
},
{
"additionalInfo": {
"layoutX": 347,
"layoutY": 149
},
"type": "org.thingsboard.rule.engine.filter.TbMsgTypeSwitchNode",
"name": "Message Type Switch",
"debugMode": false,
"configuration": {
"version": 0
}
},
{
"additionalInfo": {
"layoutX": 825,
"layoutY": 266
},
"type": "org.thingsboard.rule.engine.action.TbLogNode",
"name": "Log RPC from Device",
"debugMode": false,
"configuration": {
"jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);"
}
},
{
"additionalInfo": {
"layoutX": 825,
"layoutY": 379
},
"type": "org.thingsboard.rule.engine.action.TbLogNode",
"name": "Log Other",
"debugMode": false,
"configuration": {
"jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);"
}
},
{
"additionalInfo": {
"layoutX": 825,
"layoutY": 468
},
"type": "org.thingsboard.rule.engine.rpc.TbSendRPCRequestNode",
"name": "RPC Call Request",
"debugMode": false,
"configuration": {
"timeoutInSeconds": 60
}
},
{
"additionalInfo": {
"layoutX": 1069,
"layoutY": 90
},
"type": "org.thingsboard.rule.engine.filter.TbJsFilterNode",
"name": "Is Thermostat?",
"debugMode": false,
"configuration": {
"jsScript": "return metadata[\"deviceType\"] === \"thermostat\";"
}
}
],
"connections": [
{
"fromIndex": 0,
"toIndex": 6,
"type": "Success"
},
{
"fromIndex": 2,
"toIndex": 4,
"type": "Other"
},
{
"fromIndex": 2,
"toIndex": 1,
"type": "Post attributes"
},
{
"fromIndex": 2,
"toIndex": 0,
"type": "Post telemetry"
},
{
"fromIndex": 2,
"toIndex": 3,
"type": "RPC Request from Device"
},
{
"fromIndex": 2,
"toIndex": 5,
"type": "RPC Request to Device"
}
],
"ruleChainConnections": [
{
"fromIndex": 6,
"targetRuleChainId": {
"entityType": "RULE_CHAIN",
"id": "83d42540-85fd-11ea-aee2-794850541ced"
},
"additionalInfo": {
"layoutX": 1088,
"layoutY": 203,
"ruleChainNodeId": "rule-chain-node-9"
},
"type": "True"
}
]
}
}

137
application/src/main/data/json/demo/rule_chains/thermostat_alarms.json

@ -0,0 +1,137 @@
{
"ruleChain": {
"additionalInfo": null,
"name": "Thermostat Alarms",
"firstRuleNodeId": null,
"root": false,
"debugMode": false,
"configuration": null
},
"metadata": {
"firstNodeIndex": 5,
"nodes": [
{
"additionalInfo": {
"layoutX": 929,
"layoutY": 67
},
"type": "org.thingsboard.rule.engine.action.TbCreateAlarmNode",
"name": "Create Temp Alarm",
"debugMode": false,
"configuration": {
"alarmType": "High Temperature",
"alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\ndetails.triggerValue = msg.temperature;\nreturn details;",
"severity": "MAJOR",
"propagate": true,
"useMessageAlarmData": false,
"relationTypes": []
}
},
{
"additionalInfo": {
"layoutX": 930,
"layoutY": 201
},
"type": "org.thingsboard.rule.engine.action.TbClearAlarmNode",
"name": "Clear Temp Alarm",
"debugMode": false,
"configuration": {
"alarmType": "High Temperature",
"alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\nreturn details;"
}
},
{
"additionalInfo": {
"layoutX": 930,
"layoutY": 131
},
"type": "org.thingsboard.rule.engine.action.TbCreateAlarmNode",
"name": "Create Humidity Alarm",
"debugMode": false,
"configuration": {
"alarmType": "Low Humidity",
"alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\ndetails.triggerValue = msg.humidity;\nreturn details;",
"severity": "MINOR",
"propagate": true,
"useMessageAlarmData": false,
"relationTypes": []
}
},
{
"additionalInfo": {
"layoutX": 929,
"layoutY": 275
},
"type": "org.thingsboard.rule.engine.action.TbClearAlarmNode",
"name": "Clear Humidity Alarm",
"debugMode": false,
"configuration": {
"alarmType": "Low Humidity",
"alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\nreturn details;"
}
},
{
"additionalInfo": {
"layoutX": 586,
"layoutY": 148
},
"type": "org.thingsboard.rule.engine.filter.TbJsSwitchNode",
"name": "Check Alarms",
"debugMode": true,
"configuration": {
"jsScript": "var relations = [];\nif(metadata[\"ss_alarmTemperature\"] === \"true\"){\n if(msg.temperature > metadata[\"ss_thresholdTemperature\"]){\n relations.push(\"NewTempAlarm\");\n } else {\n relations.push(\"ClearTempAlarm\");\n }\n}\nif(metadata[\"ss_alarmHumidity\"] === \"true\"){\n if(msg.humidity < metadata[\"ss_thresholdHumidity\"]){\n relations.push(\"NewHumidityAlarm\");\n } else {\n relations.push(\"ClearHumidityAlarm\");\n }\n}\n\nreturn relations;"
}
},
{
"additionalInfo": {
"layoutX": 321,
"layoutY": 149
},
"type": "org.thingsboard.rule.engine.metadata.TbGetAttributesNode",
"name": "Fetch Configuration",
"debugMode": true,
"configuration": {
"clientAttributeNames": [],
"sharedAttributeNames": [],
"serverAttributeNames": [
"alarmTemperature",
"thresholdTemperature",
"alarmHumidity",
"thresholdHumidity"
],
"latestTsKeyNames": [],
"tellFailureIfAbsent": false,
"getLatestValueWithTs": false
}
}
],
"connections": [
{
"fromIndex": 4,
"toIndex": 0,
"type": "NewTempAlarm"
},
{
"fromIndex": 4,
"toIndex": 1,
"type": "ClearTempAlarm"
},
{
"fromIndex": 4,
"toIndex": 2,
"type": "NewHumidityAlarm"
},
{
"fromIndex": 4,
"toIndex": 3,
"type": "ClearHumidityAlarm"
},
{
"fromIndex": 5,
"toIndex": 4,
"type": "Success"
}
],
"ruleChainConnections": null
}
}

32
application/src/main/data/json/system/widget_bundles/gateway_widgets.json

File diff suppressed because one or more lines are too long

25
application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql

@ -123,3 +123,28 @@ BEGIN
END LOOP; END LOOP;
END END
$$; $$;
CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint)
LANGUAGE plpgsql AS
$$
DECLARE
ttl_ts bigint;
debug_ttl_ts bigint;
ttl_deleted_count bigint DEFAULT 0;
debug_ttl_deleted_count bigint DEFAULT 0;
BEGIN
IF ttl > 0 THEN
ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint;
EXECUTE format(
'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count;
END IF;
IF debug_ttl > 0 THEN
debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint;
EXECUTE format(
'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count;
END IF;
RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count;
RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count;
deleted := ttl_deleted_count + debug_ttl_deleted_count;
END
$$;

3
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -32,6 +32,7 @@ import lombok.Setter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.data.redis.core.RedisTemplate; import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@ -233,6 +234,7 @@ public class ActorSystemContext {
/** /**
* The following Service will be null if we operate in tb-core mode * The following Service will be null if we operate in tb-core mode
*/ */
@Lazy
@Autowired(required = false) @Autowired(required = false)
@Getter @Getter
private TbRuleEngineDeviceRpcService tbRuleEngineDeviceRpcService; private TbRuleEngineDeviceRpcService tbRuleEngineDeviceRpcService;
@ -240,6 +242,7 @@ public class ActorSystemContext {
/** /**
* The following Service will be null if we operate in tb-rule-engine mode * The following Service will be null if we operate in tb-rule-engine mode
*/ */
@Lazy
@Autowired(required = false) @Autowired(required = false)
@Getter @Getter
private TbCoreDeviceRpcService tbCoreDeviceRpcService; private TbCoreDeviceRpcService tbCoreDeviceRpcService;

3
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java

@ -192,6 +192,9 @@ class DefaultTbContext implements TbContext {
@Override @Override
public void ack(TbMsg tbMsg) { public void ack(TbMsg tbMsg) {
if (nodeCtx.getSelf().isDebugMode()) {
mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), tbMsg, "ACK", null);
}
tbMsg.getCallback().onSuccess(); tbMsg.getCallback().onSuccess();
} }

5
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java

@ -249,7 +249,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
if (relationsCount == 0) { if (relationsCount == 0) {
log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId()); log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId());
if (relationTypes.contains(TbRelationTypes.FAILURE)) { if (relationTypes.contains(TbRelationTypes.FAILURE)) {
RuleNodeCtx ruleNodeCtx = nodeActors.get(originatorNodeId); RuleNodeCtx ruleNodeCtx = nodeActors.get(originatorNodeId);
if (ruleNodeCtx != null) { if (ruleNodeCtx != null) {
msg.getCallback().onFailure(new RuleNodeException(failureMessage, ruleChainName, ruleNodeCtx.getSelf())); msg.getCallback().onFailure(new RuleNodeException(failureMessage, ruleChainName, ruleNodeCtx.getSelf()));
} else { } else {
@ -327,6 +327,9 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
private void pushMsgToNode(RuleNodeCtx nodeCtx, TbMsg msg, String fromRelationType) { private void pushMsgToNode(RuleNodeCtx nodeCtx, TbMsg msg, String fromRelationType) {
if (nodeCtx != null) { if (nodeCtx != null) {
nodeCtx.getSelfActor().tell(new RuleChainToRuleNodeMsg(new DefaultTbContext(systemContext, nodeCtx), msg, fromRelationType), self); nodeCtx.getSelfActor().tell(new RuleChainToRuleNodeMsg(new DefaultTbContext(systemContext, nodeCtx), msg, fromRelationType), self);
} else {
log.error("[{}][{}] RuleNodeCtx is empty", entityId, ruleChainName);
msg.getCallback().onFailure(new RuleEngineException("Rule Node CTX is empty"));
} }
} }

2
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java

@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.common.msg.queue.PartitionChangeMsg;
public class RuleNodeActor extends ComponentActor<RuleNodeId, RuleNodeActorMessageProcessor> { public class RuleNodeActor extends ComponentActor<RuleNodeId, RuleNodeActorMessageProcessor> {
@ -54,6 +55,7 @@ public class RuleNodeActor extends ComponentActor<RuleNodeId, RuleNodeActorMessa
onStatsPersistTick(id); onStatsPersistTick(id);
break; break;
case PARTITION_CHANGE_MSG: case PARTITION_CHANGE_MSG:
onClusterEventMsg((PartitionChangeMsg) msg);
break; break;
default: default:
return false; return false;

40
application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java

@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.msg.MsgType;
import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg;
@ -73,24 +74,30 @@ public class TenantActor extends RuleChainManagerActor {
log.info("[{}] Starting tenant actor.", tenantId); log.info("[{}] Starting tenant actor.", tenantId);
try { try {
Tenant tenant = systemContext.getTenantService().findTenantById(tenantId); Tenant tenant = systemContext.getTenantService().findTenantById(tenantId);
// This Service may be started for specific tenant only. if (tenant == null) {
Optional<TenantId> isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); cantFindTenant = true;
log.info("[{}] Started tenant actor for missing tenant.", tenantId);
} else {
// This Service may be started for specific tenant only.
Optional<TenantId> isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant();
isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE);
isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE);
if (isRuleEngineForCurrentTenant) { if (isRuleEngineForCurrentTenant) {
try { try {
if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) { if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) {
initRuleChains(); log.info("[{}] Going to init rule chains", tenantId);
} else { initRuleChains();
isRuleEngineForCurrentTenant = false; } else {
isRuleEngineForCurrentTenant = false;
}
} catch (Exception e) {
cantFindTenant = true;
} }
} catch (Exception e) {
cantFindTenant = true;
} }
log.info("[{}] Tenant actor started.", tenantId);
} }
log.info("[{}] Tenant actor started.", tenantId);
} catch (Exception e) { } catch (Exception e) {
log.warn("[{}] Unknown failure", tenantId, e); log.warn("[{}] Unknown failure", tenantId, e);
} }
@ -104,7 +111,12 @@ public class TenantActor extends RuleChainManagerActor {
@Override @Override
protected boolean process(TbActorMsg msg) { protected boolean process(TbActorMsg msg) {
if (cantFindTenant) { if (cantFindTenant) {
log.info("Missing Tenant msg: {}", msg); log.info("[{}] Processing missing Tenant msg: {}", tenantId, msg);
if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) {
QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg;
queueMsg.getTbMsg().getCallback().onSuccess();
}
return true;
} }
switch (msg.getMsgType()) { switch (msg.getMsgType()) {
case PARTITION_CHANGE_MSG: case PARTITION_CHANGE_MSG:

2
application/src/main/java/org/thingsboard/server/controller/QueueController.java

@ -43,7 +43,7 @@ public class QueueController extends BaseController {
ServiceType type = ServiceType.valueOf(serviceType); ServiceType type = ServiceType.valueOf(serviceType);
switch (type) { switch (type) {
case TB_RULE_ENGINE: case TB_RULE_ENGINE:
return Arrays.asList("HighPriority", "Main"); return Arrays.asList("Main", "HighPriority", "SequentialByOriginator");
default: default:
return Collections.emptyList(); return Collections.emptyList();
} }

14
application/src/main/java/org/thingsboard/server/controller/RuleChainController.java

@ -161,15 +161,15 @@ public class RuleChainController extends BaseController {
TenantId tenantId = getCurrentUser().getTenantId(); TenantId tenantId = getCurrentUser().getTenantId();
RuleChain previousRootRuleChain = ruleChainService.getRootTenantRuleChain(tenantId); RuleChain previousRootRuleChain = ruleChainService.getRootTenantRuleChain(tenantId);
if (ruleChainService.setRootRuleChain(getTenantId(), ruleChainId)) { if (ruleChainService.setRootRuleChain(getTenantId(), ruleChainId)) {
if (previousRootRuleChain != null) {
previousRootRuleChain = ruleChainService.findRuleChainById(getTenantId(), previousRootRuleChain.getId());
previousRootRuleChain = ruleChainService.findRuleChainById(getTenantId(), previousRootRuleChain.getId()); tbClusterService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(),
ComponentLifecycleEvent.UPDATED);
tbClusterService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(),
ComponentLifecycleEvent.UPDATED);
logEntityAction(previousRootRuleChain.getId(), previousRootRuleChain,
null, ActionType.UPDATED, null);
logEntityAction(previousRootRuleChain.getId(), previousRootRuleChain,
null, ActionType.UPDATED, null);
}
ruleChain = ruleChainService.findRuleChainById(getTenantId(), ruleChainId); ruleChain = ruleChainService.findRuleChainById(getTenantId(), ruleChainId);
tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(),

11
application/src/main/java/org/thingsboard/server/controller/TelemetryController.java

@ -344,7 +344,7 @@ public class TelemetryController extends BaseController {
return deleteAttributes(entityId, scope, keysStr); return deleteAttributes(entityId, scope, keysStr);
} }
private DeferredResult<ResponseEntity> deleteAttributes(EntityId entityIdStr, String scope, String keysStr) throws ThingsboardException { private DeferredResult<ResponseEntity> deleteAttributes(EntityId entityIdSrc, String scope, String keysStr) throws ThingsboardException {
List<String> keys = toKeysList(keysStr); List<String> keys = toKeysList(keysStr);
if (keys.isEmpty()) { if (keys.isEmpty()) {
return getImmediateDeferredResult("Empty keys: " + keysStr, HttpStatus.BAD_REQUEST); return getImmediateDeferredResult("Empty keys: " + keysStr, HttpStatus.BAD_REQUEST);
@ -354,13 +354,13 @@ public class TelemetryController extends BaseController {
if (DataConstants.SERVER_SCOPE.equals(scope) || if (DataConstants.SERVER_SCOPE.equals(scope) ||
DataConstants.SHARED_SCOPE.equals(scope) || DataConstants.SHARED_SCOPE.equals(scope) ||
DataConstants.CLIENT_SCOPE.equals(scope)) { DataConstants.CLIENT_SCOPE.equals(scope)) {
return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdStr, (result, tenantId, entityId) -> { return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdSrc, (result, tenantId, entityId) -> {
ListenableFuture<List<Void>> future = attributesService.removeAll(user.getTenantId(), entityId, scope, keys); ListenableFuture<List<Void>> future = attributesService.removeAll(user.getTenantId(), entityId, scope, keys);
Futures.addCallback(future, new FutureCallback<List<Void>>() { Futures.addCallback(future, new FutureCallback<List<Void>>() {
@Override @Override
public void onSuccess(@Nullable List<Void> tmp) { public void onSuccess(@Nullable List<Void> tmp) {
logAttributesDeleted(user, entityId, scope, keys, null); logAttributesDeleted(user, entityId, scope, keys, null);
if (entityId.getEntityType() == EntityType.DEVICE) { if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) {
DeviceId deviceId = new DeviceId(entityId.getId()); DeviceId deviceId = new DeviceId(entityId.getId());
Set<AttributeKey> keysToNotify = new HashSet<>(); Set<AttributeKey> keysToNotify = new HashSet<>();
keys.forEach(key -> keysToNotify.add(new AttributeKey(scope, key))); keys.forEach(key -> keysToNotify.add(new AttributeKey(scope, key)));
@ -397,11 +397,6 @@ public class TelemetryController extends BaseController {
@Override @Override
public void onSuccess(@Nullable Void tmp) { public void onSuccess(@Nullable Void tmp) {
logAttributesUpdated(user, entityId, scope, attributes, null); logAttributesUpdated(user, entityId, scope, attributes, null);
if (entityId.getEntityType() == EntityType.DEVICE) {
DeviceId deviceId = new DeviceId(entityId.getId());
tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate(
user.getTenantId(), deviceId, scope, attributes), null);
}
result.setResult(new ResponseEntity(HttpStatus.OK)); result.setResult(new ResponseEntity(HttpStatus.OK));
} }

2
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@ -23,8 +23,8 @@ import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Profile; import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.service.install.DatabaseTsUpgradeService;
import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService; import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService;
import org.thingsboard.server.service.install.DatabaseTsUpgradeService;
import org.thingsboard.server.service.install.EntityDatabaseSchemaService; import org.thingsboard.server.service.install.EntityDatabaseSchemaService;
import org.thingsboard.server.service.install.SystemDataLoaderService; import org.thingsboard.server.service.install.SystemDataLoaderService;
import org.thingsboard.server.service.install.TsDatabaseSchemaService; import org.thingsboard.server.service.install.TsDatabaseSchemaService;

34
application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java

@ -25,24 +25,32 @@ import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
import org.thingsboard.server.common.data.kv.BooleanDataEntry;
import org.thingsboard.server.common.data.kv.DoubleDataEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceCredentialsService;
import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.dao.widget.WidgetsBundleService;
import java.util.Arrays;
@Service @Service
@Profile("install") @Profile("install")
@Slf4j @Slf4j
@ -76,6 +84,9 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
@Autowired @Autowired
private DeviceService deviceService; private DeviceService deviceService;
@Autowired
private AttributesService attributesService;
@Autowired @Autowired
private DeviceCredentialsService deviceCredentialsService; private DeviceCredentialsService deviceCredentialsService;
@ -120,7 +131,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
demoTenant.setRegion("Global"); demoTenant.setRegion("Global");
demoTenant.setTitle("Tenant"); demoTenant.setTitle("Tenant");
demoTenant = tenantService.saveTenant(demoTenant); demoTenant = tenantService.saveTenant(demoTenant);
installScripts.createDefaultRuleChains(demoTenant.getId()); installScripts.loadDemoRuleChains(demoTenant.getId());
createUser(Authority.TENANT_ADMIN, demoTenant.getId(), null, "tenant@thingsboard.org", "tenant"); createUser(Authority.TENANT_ADMIN, demoTenant.getId(), null, "tenant@thingsboard.org", "tenant");
Customer customerA = new Customer(); Customer customerA = new Customer();
@ -152,6 +163,25 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
createDevice(demoTenant.getId(), null, DEFAULT_DEVICE_TYPE, "Raspberry Pi Demo Device", "RASPBERRY_PI_DEMO_TOKEN", "Demo device that is used in " + createDevice(demoTenant.getId(), null, DEFAULT_DEVICE_TYPE, "Raspberry Pi Demo Device", "RASPBERRY_PI_DEMO_TOKEN", "Demo device that is used in " +
"Raspberry Pi GPIO control sample application"); "Raspberry Pi GPIO control sample application");
DeviceId t1Id = createDevice(demoTenant.getId(), null, "thermostat", "Thermostat T1", "T1_TEST_TOKEN", "Demo device for Thermostats dashboard").getId();
DeviceId t2Id = createDevice(demoTenant.getId(), null, "thermostat", "Thermostat T2", "T2_TEST_TOKEN", "Demo device for Thermostats dashboard").getId();
attributesService.save(demoTenant.getId(), t1Id, DataConstants.SERVER_SCOPE,
Arrays.asList(new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 37.3948)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", -122.1503)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmTemperature", true)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmHumidity", true)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdTemperature", (long) 20)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdHumidity", (long) 50))));
attributesService.save(demoTenant.getId(), t2Id, DataConstants.SERVER_SCOPE,
Arrays.asList(new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 37.493801)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", -121.948769)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmTemperature", true)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmHumidity", true)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdTemperature", (long) 25)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdHumidity", (long) 30))));
installScripts.loadDashboards(demoTenant.getId(), null); installScripts.loadDashboards(demoTenant.getId(), null);
} }

27
application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java

@ -24,6 +24,7 @@ import org.springframework.util.StringUtils;
import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainMetaData;
@ -182,4 +183,30 @@ public class InstallScripts {
} }
public void loadDemoRuleChains(TenantId tenantId) throws Exception {
Path ruleChainsDir = Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, RULE_CHAINS_DIR);
try {
JsonNode ruleChainJson = objectMapper.readTree(ruleChainsDir.resolve("thermostat_alarms.json").toFile());
RuleChain ruleChain = objectMapper.treeToValue(ruleChainJson.get("ruleChain"), RuleChain.class);
RuleChainMetaData ruleChainMetaData = objectMapper.treeToValue(ruleChainJson.get("metadata"), RuleChainMetaData.class);
ruleChain.setTenantId(tenantId);
ruleChain = ruleChainService.saveRuleChain(ruleChain);
ruleChainMetaData.setRuleChainId(ruleChain.getId());
ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), ruleChainMetaData);
JsonNode rootChainJson = objectMapper.readTree(ruleChainsDir.resolve("root_rule_chain.json").toFile());
RuleChain rootChain = objectMapper.treeToValue(rootChainJson.get("ruleChain"), RuleChain.class);
RuleChainMetaData rootChainMetaData = objectMapper.treeToValue(rootChainJson.get("metadata"), RuleChainMetaData.class);
RuleChainId thermostatsRuleChainId = ruleChain.getId();
rootChainMetaData.getRuleChainConnections().forEach(connection -> connection.setTargetRuleChainId(thermostatsRuleChainId));
rootChain.setTenantId(tenantId);
rootChain = ruleChainService.saveRuleChain(rootChain);
rootChainMetaData.setRuleChainId(rootChain.getId());
ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), rootChainMetaData);
} catch (Exception e) {
log.error("Unable to load dashboard from json", e);
throw new RuntimeException("Unable to load dashboard from json", e);
}
}
} }

5
application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java

@ -225,6 +225,11 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
conn.createStatement().execute("ALTER TABLE tenant ADD COLUMN isolated_tb_core boolean DEFAULT (false), ADD COLUMN isolated_tb_rule_engine boolean DEFAULT (false)"); conn.createStatement().execute("ALTER TABLE tenant ADD COLUMN isolated_tb_core boolean DEFAULT (false), ADD COLUMN isolated_tb_rule_engine boolean DEFAULT (false)");
} catch (Exception e) { } catch (Exception e) {
} }
try {
long ts = System.currentTimeMillis();
conn.createStatement().execute("ALTER TABLE event ADD COLUMN ts bigint DEFAULT " + ts + ";"); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
} catch (Exception e) {
}
log.info("Schema updated."); log.info("Schema updated.");
} }
break; break;

6
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java

@ -87,6 +87,7 @@ public class DefaultTbClusterService implements TbClusterService {
@Override @Override
public void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback) { public void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback) {
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId());
log.trace("PUSHING msg: {} to:{}", msg, tpi);
byte[] msgBytes = encodingService.encode(msg); byte[] msgBytes = encodingService.encode(msg);
ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToDeviceActorNotificationMsg(ByteString.copyFrom(msgBytes)).build(); ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToDeviceActorNotificationMsg(ByteString.copyFrom(msgBytes)).build();
producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msg.getDeviceId().getId(), toCoreMsg), callback); producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msg.getDeviceId().getId(), toCoreMsg), callback);
@ -96,6 +97,7 @@ public class DefaultTbClusterService implements TbClusterService {
@Override @Override
public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId);
log.trace("PUSHING msg: {} to:{}", response, tpi);
FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder()
.setRequestIdMSB(response.getId().getMostSignificantBits()) .setRequestIdMSB(response.getId().getMostSignificantBits())
.setRequestIdLSB(response.getId().getLeastSignificantBits()) .setRequestIdLSB(response.getId().getLeastSignificantBits())
@ -108,6 +110,7 @@ public class DefaultTbClusterService implements TbClusterService {
@Override @Override
public void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, ToRuleEngineMsg msg, TbQueueCallback callback) { public void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, ToRuleEngineMsg msg, TbQueueCallback callback) {
log.trace("PUSHING msg: {} to:{}", msg, tpi);
producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback);
toRuleEngineMsgs.incrementAndGet(); toRuleEngineMsgs.incrementAndGet();
} }
@ -123,6 +126,7 @@ public class DefaultTbClusterService implements TbClusterService {
} }
} }
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId);
log.trace("PUSHING msg: {} to:{}", tbMsg, tpi);
ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
@ -134,6 +138,7 @@ public class DefaultTbClusterService implements TbClusterService {
@Override @Override
public void pushNotificationToRuleEngine(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { public void pushNotificationToRuleEngine(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId);
log.trace("PUSHING msg: {} to:{}", response, tpi);
FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder()
.setRequestIdMSB(response.getId().getMostSignificantBits()) .setRequestIdMSB(response.getId().getMostSignificantBits())
.setRequestIdLSB(response.getId().getLeastSignificantBits()) .setRequestIdLSB(response.getId().getLeastSignificantBits())
@ -147,6 +152,7 @@ public class DefaultTbClusterService implements TbClusterService {
@Override @Override
public void pushNotificationToTransport(String serviceId, ToTransportMsg response, TbQueueCallback callback) { public void pushNotificationToTransport(String serviceId, ToTransportMsg response, TbQueueCallback callback) {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceId); TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceId);
log.trace("PUSHING msg: {} to:{}", response, tpi);
producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), response), callback); producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), response), callback);
toTransportNfs.incrementAndGet(); toTransportNfs.incrementAndGet();
} }

15
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -52,6 +52,7 @@ import org.thingsboard.server.service.subscription.TbSubscriptionUtils;
import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.UUID; import java.util.UUID;
@ -98,6 +99,11 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
super.init("tb-core-consumer", "tb-core-notifications-consumer"); super.init("tb-core-consumer", "tb-core-notifications-consumer");
} }
@PreDestroy
public void destroy(){
super.destroy();
}
@Override @Override
public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
if (partitionChangeEvent.getServiceType().equals(getServiceType())) { if (partitionChangeEvent.getServiceType().equals(getServiceType())) {
@ -117,11 +123,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
} }
ConcurrentMap<UUID, TbProtoQueueMsg<ToCoreMsg>> pendingMap = msgs.stream().collect( ConcurrentMap<UUID, TbProtoQueueMsg<ToCoreMsg>> pendingMap = msgs.stream().collect(
Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity()));
ConcurrentMap<UUID, TbProtoQueueMsg<ToCoreMsg>> failedMap = new ConcurrentHashMap<>();
CountDownLatch processingTimeoutLatch = new CountDownLatch(1); CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
TbPackProcessingContext<TbProtoQueueMsg<ToCoreMsg>> ctx = new TbPackProcessingContext<>(
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
pendingMap.forEach((id, msg) -> { pendingMap.forEach((id, msg) -> {
log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); log.trace("[{}] Creating main callback for message: {}", id, msg.getValue());
TbCallback callback = new TbPackCallback<>(id, processingTimeoutLatch, pendingMap, failedMap); TbCallback callback = new TbPackCallback<>(id, ctx);
try { try {
ToCoreMsg toCoreMsg = msg.getValue(); ToCoreMsg toCoreMsg = msg.getValue();
if (toCoreMsg.hasToSubscriptionMgrMsg()) { if (toCoreMsg.hasToSubscriptionMgrMsg()) {
@ -147,8 +154,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
} }
}); });
if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) {
pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue()));
failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue()));
} }
mainConsumer.commit(); mainConsumer.commit();
} catch (Exception e) { } catch (Exception e) {

10
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@ -31,7 +31,6 @@ import org.thingsboard.server.common.msg.queue.ServiceQueue;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
@ -64,9 +63,6 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.stream.Collectors;
@Service @Service
@TbRuleEngineComponent @TbRuleEngineComponent
@ -116,10 +112,10 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
@PreDestroy @PreDestroy
public void stop() { public void stop() {
super.destroy();
if (submitExecutor != null) { if (submitExecutor != null) {
submitExecutor.shutdownNow(); submitExecutor.shutdownNow();
} }
ruleEngineSettings.getQueues().forEach(config -> consumerConfigurations.put(config.getName(), config)); ruleEngineSettings.getQueues().forEach(config -> consumerConfigurations.put(config.getName(), config));
} }
@ -156,7 +152,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
submitStrategy.init(msgs); submitStrategy.init(msgs);
while (!stopped) { while (!stopped) {
ProcessingAttemptContext ctx = new ProcessingAttemptContext(submitStrategy); TbMsgPackProcessingContext ctx = new TbMsgPackProcessingContext(submitStrategy);
submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> { submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> {
log.trace("[{}] Creating callback for message: {}", id, msg.getValue()); log.trace("[{}] Creating callback for message: {}", id, msg.getValue());
ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); ToRuleEngineMsg toRuleEngineMsg = msg.getValue();
@ -174,7 +170,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
})); }));
boolean timeout = false; boolean timeout = false;
if (!ctx.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { if (!ctx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS)) {
timeout = true; timeout = true;
} }

2
application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java

@ -79,7 +79,7 @@ public class TbCoreConsumerStats {
public void printStats() { public void printStats() {
int total = totalCounter.getAndSet(0); int total = totalCounter.getAndSet(0);
if (total > 0) { if (total > 0) {
log.info("Transport total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]" + log.info("Total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]" +
" deviceState [{}] subMgr [{}] coreNfs [{}]", " deviceState [{}] subMgr [{}] coreNfs [{}]",
total, sessionEventCounter.getAndSet(0), total, sessionEventCounter.getAndSet(0),
getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0),

8
application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java

@ -18,21 +18,17 @@ package org.thingsboard.server.service.queue;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.common.msg.queue.RuleNodeException;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
@Slf4j @Slf4j
public class TbMsgPackCallback implements TbMsgCallback { public class TbMsgPackCallback implements TbMsgCallback {
private final UUID id; private final UUID id;
private final TenantId tenantId; private final TenantId tenantId;
private final ProcessingAttemptContext ctx; private final TbMsgPackProcessingContext ctx;
public TbMsgPackCallback(UUID id, TenantId tenantId, ProcessingAttemptContext ctx) { public TbMsgPackCallback(UUID id, TenantId tenantId, TbMsgPackProcessingContext ctx) {
this.id = id; this.id = id;
this.tenantId = tenantId; this.tenantId = tenantId;
this.ctx = ctx; this.ctx = ctx;

25
application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java → application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java

@ -27,11 +27,13 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch; import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
public class ProcessingAttemptContext { public class TbMsgPackProcessingContext {
private final TbRuleEngineSubmitStrategy submitStrategy; private final TbRuleEngineSubmitStrategy submitStrategy;
private final AtomicInteger pendingCount;
private final CountDownLatch processingTimeoutLatch = new CountDownLatch(1); private final CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
@Getter @Getter
private final ConcurrentMap<UUID, TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> pendingMap; private final ConcurrentMap<UUID, TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> pendingMap;
@ -42,9 +44,10 @@ public class ProcessingAttemptContext {
@Getter @Getter
private final ConcurrentMap<TenantId, RuleEngineException> exceptionsMap = new ConcurrentHashMap<>(); private final ConcurrentMap<TenantId, RuleEngineException> exceptionsMap = new ConcurrentHashMap<>();
public ProcessingAttemptContext(TbRuleEngineSubmitStrategy submitStrategy) { public TbMsgPackProcessingContext(TbRuleEngineSubmitStrategy submitStrategy) {
this.submitStrategy = submitStrategy; this.submitStrategy = submitStrategy;
this.pendingMap = submitStrategy.getPendingMap(); this.pendingMap = submitStrategy.getPendingMap();
this.pendingCount = new AtomicInteger(pendingMap.size());
} }
public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException { public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException {
@ -54,16 +57,12 @@ public class ProcessingAttemptContext {
public void onSuccess(UUID id) { public void onSuccess(UUID id) {
TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg> msg; TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg> msg;
boolean empty = false; boolean empty = false;
synchronized (pendingMap) { msg = pendingMap.remove(id);
msg = pendingMap.remove(id);
if (msg != null) {
empty = pendingMap.isEmpty();
}
}
if (msg != null) { if (msg != null) {
empty = pendingCount.decrementAndGet() == 0;
successMap.put(id, msg); successMap.put(id, msg);
submitStrategy.onSuccess(id);
} }
submitStrategy.onSuccess(id);
if (empty) { if (empty) {
processingTimeoutLatch.countDown(); processingTimeoutLatch.countDown();
} }
@ -72,13 +71,9 @@ public class ProcessingAttemptContext {
public void onFailure(TenantId tenantId, UUID id, RuleEngineException e) { public void onFailure(TenantId tenantId, UUID id, RuleEngineException e) {
TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg> msg; TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg> msg;
boolean empty = false; boolean empty = false;
synchronized (pendingMap) { msg = pendingMap.remove(id);
msg = pendingMap.remove(id);
if (msg != null) {
empty = pendingMap.isEmpty();
}
}
if (msg != null) { if (msg != null) {
empty = pendingCount.decrementAndGet() == 0;
failedMap.put(id, msg); failedMap.put(id, msg);
exceptionsMap.putIfAbsent(tenantId, e); exceptionsMap.putIfAbsent(tenantId, e);
} }

28
application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java

@ -19,44 +19,26 @@ import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbCallback;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
@Slf4j @Slf4j
public class TbPackCallback<T> implements TbCallback { public class TbPackCallback<T> implements TbCallback {
private final CountDownLatch processingTimeoutLatch; private final TbPackProcessingContext<T> ctx;
private final ConcurrentMap<UUID, T> ackMap;
private final ConcurrentMap<UUID, T> failedMap;
private final UUID id; private final UUID id;
public TbPackCallback(UUID id, public TbPackCallback(UUID id, TbPackProcessingContext<T> ctx) {
CountDownLatch processingTimeoutLatch,
ConcurrentMap<UUID, T> ackMap,
ConcurrentMap<UUID, T> failedMap) {
this.id = id; this.id = id;
this.processingTimeoutLatch = processingTimeoutLatch; this.ctx = ctx;
this.ackMap = ackMap;
this.failedMap = failedMap;
} }
@Override @Override
public void onSuccess() { public void onSuccess() {
log.trace("[{}] ON SUCCESS", id); log.trace("[{}] ON SUCCESS", id);
T msg = ackMap.remove(id); ctx.onSuccess(id);
if (msg != null && ackMap.isEmpty()) {
processingTimeoutLatch.countDown();
}
} }
@Override @Override
public void onFailure(Throwable t) { public void onFailure(Throwable t) {
log.trace("[{}] ON FAILURE", id, t); log.trace("[{}] ON FAILURE", id, t);
T msg = ackMap.remove(id); ctx.onFailure(id, t);
if (msg != null) {
failedMap.put(id, msg);
}
if (ackMap.isEmpty()) {
processingTimeoutLatch.countDown();
}
} }
} }

90
application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java

@ -0,0 +1,90 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.queue;
import lombok.extern.slf4j.Slf4j;
import java.util.UUID;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@Slf4j
public class TbPackProcessingContext<T> {
private final AtomicInteger pendingCount;
private final CountDownLatch processingTimeoutLatch;
private final ConcurrentMap<UUID, T> ackMap;
private final ConcurrentMap<UUID, T> failedMap;
public TbPackProcessingContext(CountDownLatch processingTimeoutLatch,
ConcurrentMap<UUID, T> ackMap,
ConcurrentMap<UUID, T> failedMap) {
this.processingTimeoutLatch = processingTimeoutLatch;
this.pendingCount = new AtomicInteger(ackMap.size());
this.ackMap = ackMap;
this.failedMap = failedMap;
}
public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException {
return processingTimeoutLatch.await(packProcessingTimeout, milliseconds);
}
public void onSuccess(UUID id) {
boolean empty = false;
T msg = ackMap.remove(id);
if (msg != null) {
empty = pendingCount.decrementAndGet() == 0;
}
if (empty) {
processingTimeoutLatch.countDown();
} else {
if (log.isTraceEnabled()) {
log.trace("Items left: {}", ackMap.size());
for (T t : ackMap.values()) {
log.trace("left item: {}", t);
}
}
}
}
public void onFailure(UUID id, Throwable t) {
boolean empty = false;
T msg = ackMap.remove(id);
if (msg != null) {
empty = pendingCount.decrementAndGet() == 0;
failedMap.put(id, msg);
if (log.isTraceEnabled()) {
log.trace("Items left: {}", ackMap.size());
for (T v : ackMap.values()) {
log.trace("left item: {}", v);
}
}
}
if (empty) {
processingTimeoutLatch.countDown();
}
}
public ConcurrentMap<UUID, T> getAckMap() {
return ackMap;
}
public ConcurrentMap<UUID, T> getFailedMap() {
return failedMap;
}
}

11
application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java

@ -23,11 +23,13 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService;
import org.thingsboard.server.service.queue.TbPackCallback; import org.thingsboard.server.service.queue.TbPackCallback;
import org.thingsboard.server.service.queue.TbPackProcessingContext;
import javax.annotation.PreDestroy; import javax.annotation.PreDestroy;
import java.util.List; import java.util.List;
@ -92,11 +94,12 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
} }
ConcurrentMap<UUID, TbProtoQueueMsg<N>> pendingMap = msgs.stream().collect( ConcurrentMap<UUID, TbProtoQueueMsg<N>> pendingMap = msgs.stream().collect(
Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity()));
ConcurrentMap<UUID, TbProtoQueueMsg<N>> failedMap = new ConcurrentHashMap<>();
CountDownLatch processingTimeoutLatch = new CountDownLatch(1); CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
TbPackProcessingContext<TbProtoQueueMsg<N>> ctx = new TbPackProcessingContext<>(
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
pendingMap.forEach((id, msg) -> { pendingMap.forEach((id, msg) -> {
log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue()); log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue());
TbCallback callback = new TbPackCallback<>(id, processingTimeoutLatch, pendingMap, failedMap); TbCallback callback = new TbPackCallback<>(id, ctx);
try { try {
handleNotification(id, msg, callback); handleNotification(id, msg, callback);
} catch (Throwable e) { } catch (Throwable e) {
@ -105,8 +108,8 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
} }
}); });
if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) {
pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue())); ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue()));
failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue())); ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue()));
} }
nfConsumer.commit(); nfConsumer.commit();
} catch (Exception e) { } catch (Exception e) {

6
application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java

@ -20,7 +20,7 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.service.queue.ProcessingAttemptContext; import org.thingsboard.server.service.queue.TbMsgPackProcessingContext;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
@ -32,9 +32,9 @@ public class TbRuleEngineProcessingResult {
@Getter @Getter
private final boolean timeout; private final boolean timeout;
@Getter @Getter
private final ProcessingAttemptContext ctx; private final TbMsgPackProcessingContext ctx;
public TbRuleEngineProcessingResult(boolean timeout, ProcessingAttemptContext ctx) { public TbRuleEngineProcessingResult(boolean timeout, TbMsgPackProcessingContext ctx) {
this.timeout = timeout; this.timeout = timeout;
this.ctx = ctx; this.ctx = ctx;
this.success = !timeout && ctx.getPendingMap().isEmpty() && ctx.getFailedMap().isEmpty(); this.success = !timeout && ctx.getPendingMap().isEmpty() && ctx.getFailedMap().isEmpty();

16
application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java

@ -17,6 +17,8 @@ package org.thingsboard.server.service.queue.processing;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueAckStrategyConfiguration; import org.thingsboard.server.queue.settings.TbRuleEngineQueueAckStrategyConfiguration;
@ -32,7 +34,7 @@ public class TbRuleEngineProcessingStrategyFactory {
public TbRuleEngineProcessingStrategy newInstance(String name, TbRuleEngineQueueAckStrategyConfiguration configuration) { public TbRuleEngineProcessingStrategy newInstance(String name, TbRuleEngineQueueAckStrategyConfiguration configuration) {
switch (configuration.getType()) { switch (configuration.getType()) {
case "SKIP_ALL": case "SKIP_ALL_FAILURES":
return new SkipStrategy(name); return new SkipStrategy(name);
case "RETRY_ALL": case "RETRY_ALL":
return new RetryStrategy(name, true, true, true, configuration); return new RetryStrategy(name, true, true, true, configuration);
@ -98,7 +100,7 @@ public class TbRuleEngineProcessingStrategyFactory {
} }
log.info("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); log.info("[{}] Going to reprocess {} messages", queueName, toReprocess.size());
if (log.isTraceEnabled()) { if (log.isTraceEnabled()) {
toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, msg.getValue())); toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
} }
if (pauseBetweenRetries > 0) { if (pauseBetweenRetries > 0) {
try { try {
@ -123,7 +125,15 @@ public class TbRuleEngineProcessingStrategyFactory {
@Override @Override
public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) {
log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); if (!result.isSuccess()) {
log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size());
}
if (log.isTraceEnabled()) {
result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
}
if (log.isTraceEnabled()) {
result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
}
return new TbRuleEngineProcessingDecision(true, null); return new TbRuleEngineProcessingDecision(true, null);
} }
} }

4
application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java

@ -29,9 +29,9 @@ public class TbRuleEngineSubmitStrategyFactory {
return new BurstTbRuleEngineSubmitStrategy(name); return new BurstTbRuleEngineSubmitStrategy(name);
case "BATCH": case "BATCH":
return new BatchTbRuleEngineSubmitStrategy(name, configuration.getBatchSize()); return new BatchTbRuleEngineSubmitStrategy(name, configuration.getBatchSize());
case "SEQUENTIAL_WITHIN_ORIGINATOR": case "SEQUENTIAL_BY_ORIGINATOR":
return new SequentialByOriginatorIdTbRuleEngineSubmitStrategy(name); return new SequentialByOriginatorIdTbRuleEngineSubmitStrategy(name);
case "SEQUENTIAL_WITHIN_TENANT": case "SEQUENTIAL_BY_TENANT":
return new SequentialByTenantIdTbRuleEngineSubmitStrategy(name); return new SequentialByTenantIdTbRuleEngineSubmitStrategy(name);
case "SEQUENTIAL": case "SEQUENTIAL":
return new SequentialTbRuleEngineSubmitStrategy(name); return new SequentialTbRuleEngineSubmitStrategy(name);

10
application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java

@ -106,7 +106,7 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService {
@Override @Override
public void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response) { public void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response) {
log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId()); log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), response);
UUID requestId = response.getId(); UUID requestId = response.getId();
Consumer<FromDeviceRpcResponse> consumer = localToRuleEngineRpcRequests.remove(requestId); Consumer<FromDeviceRpcResponse> consumer = localToRuleEngineRpcRequests.remove(requestId);
if (consumer != null) { if (consumer != null) {
@ -177,9 +177,9 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService {
private void scheduleToRuleEngineTimeout(ToDeviceRpcRequest request, UUID requestId) { private void scheduleToRuleEngineTimeout(ToDeviceRpcRequest request, UUID requestId) {
long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis());
log.trace("[{}] processing to rule engine request: [{}]", this.hashCode(), requestId); log.trace("[{}] processing to rule engine request.", requestId);
scheduler.schedule(() -> { scheduler.schedule(() -> {
log.trace("[{}] timeout for to rule engine request: [{}]", this.hashCode(), requestId); log.trace("[{}] timeout for processing to rule engine request.", requestId);
Consumer<FromDeviceRpcResponse> consumer = localToRuleEngineRpcRequests.remove(requestId); Consumer<FromDeviceRpcResponse> consumer = localToRuleEngineRpcRequests.remove(requestId);
if (consumer != null) { if (consumer != null) {
consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT)); consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT));
@ -189,9 +189,9 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService {
private void scheduleToDeviceTimeout(ToDeviceRpcRequest request, UUID requestId) { private void scheduleToDeviceTimeout(ToDeviceRpcRequest request, UUID requestId) {
long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis());
log.trace("[{}] processing to device request: [{}]", this.hashCode(), requestId); log.trace("[{}] processing to device request.", requestId);
scheduler.schedule(() -> { scheduler.schedule(() -> {
log.trace("[{}] timeout for to device request: [{}]", this.hashCode(), requestId); log.trace("[{}] timeout for to device request.", requestId);
localToDeviceRpcRequests.remove(requestId); localToDeviceRpcRequests.remove(requestId);
}, timeout, TimeUnit.MILLISECONDS); }, timeout, TimeUnit.MILLISECONDS);
} }

8
application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java

@ -129,10 +129,10 @@ public class DefaultDeviceStateService implements DeviceStateService {
private volatile boolean clusterUpdatePending = false; private volatile boolean clusterUpdatePending = false;
private ListeningScheduledExecutorService queueExecutor; private ListeningScheduledExecutorService queueExecutor;
private ConcurrentMap<TopicPartitionInfo, Set<DeviceId>> partitionedDevices = new ConcurrentHashMap<>(); private final ConcurrentMap<TopicPartitionInfo, Set<DeviceId>> partitionedDevices = new ConcurrentHashMap<>();
private ConcurrentMap<DeviceId, DeviceStateData> deviceStates = new ConcurrentHashMap<>(); private final ConcurrentMap<DeviceId, DeviceStateData> deviceStates = new ConcurrentHashMap<>();
private ConcurrentMap<DeviceId, Long> deviceLastReportedActivity = new ConcurrentHashMap<>(); private final ConcurrentMap<DeviceId, Long> deviceLastReportedActivity = new ConcurrentHashMap<>();
private ConcurrentMap<DeviceId, Long> deviceLastSavedActivity = new ConcurrentHashMap<>(); private final ConcurrentMap<DeviceId, Long> deviceLastSavedActivity = new ConcurrentHashMap<>();
public DefaultDeviceStateService(TenantService tenantService, DeviceService deviceService, public DefaultDeviceStateService(TenantService tenantService, DeviceService deviceService,
AttributesService attributesService, TimeseriesService tsService, AttributesService attributesService, TimeseriesService tsService,

2
application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java

@ -41,6 +41,6 @@ public interface DeviceStateService extends ApplicationListener<PartitionChangeE
void onDeviceInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout); void onDeviceInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout);
void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto serverAddress, TbCallback bytes); void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto proto, TbCallback bytes);
} }

11
application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java

@ -47,6 +47,7 @@ import java.util.stream.Collectors;
@Slf4j @Slf4j
public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService { public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService {
public static final String TB_SERVICE_QUEUE = "TbServiceQueue";
public static final FutureCallback<Void> CALLBACK = new FutureCallback<Void>() { public static final FutureCallback<Void> CALLBACK = new FutureCallback<Void>() {
@Override @Override
public void onSuccess(@Nullable Void result) { public void onSuccess(@Nullable Void result) {
@ -95,7 +96,13 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS
}); });
ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> {
TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString())); TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString()));
tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); try {
tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK);
} catch (DataValidationException e2) {
if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) {
throw e2;
}
}
}); });
ruleEngineStats.reset(); ruleEngineStats.reset();
} }
@ -113,7 +120,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS
asset = new Asset(); asset = new Asset();
asset.setTenantId(tenantId); asset.setTenantId(tenantId);
asset.setName(queueName + "_" + serviceInfoProvider.getServiceId()); asset.setName(queueName + "_" + serviceInfoProvider.getServiceId());
asset.setType("TbServiceQueue"); asset.setType(TB_SERVICE_QUEUE);
asset = assetService.saveAsset(asset); asset = assetService.saveAsset(asset);
} }
assetId = asset.getId(); assetId = asset.getId();

8
application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java

@ -345,6 +345,8 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
keys.forEach(key -> subState.put(key, 0L)); keys.forEach(key -> subState.put(key, 0L));
attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs()));
TbAttributeSubscriptionScope scope = StringUtils.isEmpty(cmd.getScope()) ? TbAttributeSubscriptionScope.SERVER_SCOPE : TbAttributeSubscriptionScope.valueOf(cmd.getScope());
TbAttributeSubscription sub = TbAttributeSubscription.builder() TbAttributeSubscription sub = TbAttributeSubscription.builder()
.serviceId(serviceId) .serviceId(serviceId)
.sessionId(sessionId) .sessionId(sessionId)
@ -353,7 +355,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
.entityId(entityId) .entityId(entityId)
.allKeys(false) .allKeys(false)
.keyStates(subState) .keyStates(subState)
.scope(TbAttributeSubscriptionScope.valueOf(cmd.getScope())).build(); .scope(scope).build();
subService.addSubscription(sub); subService.addSubscription(sub);
} }
@ -440,6 +442,8 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
Map<String, Long> subState = new HashMap<>(attributesData.size()); Map<String, Long> subState = new HashMap<>(attributesData.size());
attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs()));
TbAttributeSubscriptionScope scope = StringUtils.isEmpty(cmd.getScope()) ? TbAttributeSubscriptionScope.SERVER_SCOPE : TbAttributeSubscriptionScope.valueOf(cmd.getScope());
TbAttributeSubscription sub = TbAttributeSubscription.builder() TbAttributeSubscription sub = TbAttributeSubscription.builder()
.serviceId(serviceId) .serviceId(serviceId)
.sessionId(sessionId) .sessionId(sessionId)
@ -448,7 +452,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
.entityId(entityId) .entityId(entityId)
.allKeys(true) .allKeys(true)
.keyStates(subState) .keyStates(subState)
.scope(TbAttributeSubscriptionScope.valueOf(cmd.getScope())).build(); .scope(scope).build();
subService.addSubscription(sub); subService.addSubscription(sub);
} }

40
application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java → application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java

@ -17,47 +17,27 @@ package org.thingsboard.server.service.ttl;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled; import org.thingsboard.server.dao.util.PsqlDao;
import org.thingsboard.server.dao.util.PsqlTsAnyDao;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.SQLWarning; import java.sql.SQLWarning;
import java.sql.Statement; import java.sql.Statement;
@PsqlTsAnyDao
@Slf4j
public abstract class AbstractTimeseriesCleanUpService {
@Value("${sql.ttl.ts_key_value_ttl}")
protected long systemTtl;
@Value("${sql.ttl.enabled}") @Slf4j
private boolean ttlTaskExecutionEnabled; @PsqlDao
public abstract class AbstractCleanUpService {
@Value("${spring.datasource.url}") @Value("${spring.datasource.url}")
private String dbUrl; protected String dbUrl;
@Value("${spring.datasource.username}") @Value("${spring.datasource.username}")
private String dbUserName; protected String dbUserName;
@Value("${spring.datasource.password}") @Value("${spring.datasource.password}")
private String dbPassword; protected String dbPassword;
@Scheduled(initialDelayString = "${sql.ttl.execution_interval_ms}", fixedDelayString = "${sql.ttl.execution_interval_ms}")
public void cleanUp() {
if (ttlTaskExecutionEnabled) {
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
doCleanUp(conn);
} catch (SQLException e) {
log.error("SQLException occurred during TTL task execution ", e);
}
}
}
protected abstract void doCleanUp(Connection connection);
protected long executeQuery(Connection conn, String query) { protected long executeQuery(Connection conn, String query) {
long removed = 0L; long removed = 0L;
@ -74,7 +54,7 @@ public abstract class AbstractTimeseriesCleanUpService {
return removed; return removed;
} }
private void getWarnings(Statement statement) throws SQLException { protected void getWarnings(Statement statement) throws SQLException {
SQLWarning warnings = statement.getWarnings(); SQLWarning warnings = statement.getWarnings();
if (warnings != null) { if (warnings != null) {
log.debug("{}", warnings.getMessage()); log.debug("{}", warnings.getMessage());
@ -86,4 +66,6 @@ public abstract class AbstractTimeseriesCleanUpService {
} }
} }
} protected abstract void doCleanUp(Connection connection);
}

59
application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java

@ -0,0 +1,59 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.ttl.events;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.PsqlDao;
import org.thingsboard.server.service.ttl.AbstractCleanUpService;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
@PsqlDao
@Slf4j
@Service
public class EventsCleanUpService extends AbstractCleanUpService {
@Value("${sql.ttl.events.events_ttl}")
private long ttl;
@Value("${sql.ttl.events.debug_events_ttl}")
private long debugTtl;
@Value("${sql.ttl.events.enabled}")
private boolean ttlTaskExecutionEnabled;
@Scheduled(initialDelayString = "${sql.ttl.events.execution_interval_ms}", fixedDelayString = "${sql.ttl.events.execution_interval_ms}")
public void cleanUp() {
if (ttlTaskExecutionEnabled) {
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
doCleanUp(conn);
} catch (SQLException e) {
log.error("SQLException occurred during TTL task execution ", e);
}
}
}
@Override
protected void doCleanUp(Connection connection) {
long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", " + debugTtl + ", 0);");
log.info("Total events removed by TTL: [{}]", totalEventsRemoved);
}
}

49
application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java

@ -0,0 +1,49 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.ttl.timeseries;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.thingsboard.server.dao.util.PsqlTsAnyDao;
import org.thingsboard.server.service.ttl.AbstractCleanUpService;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
@PsqlTsAnyDao
@Slf4j
public abstract class AbstractTimeseriesCleanUpService extends AbstractCleanUpService {
@Value("${sql.ttl.ts.ts_key_value_ttl}")
protected long systemTtl;
@Value("${sql.ttl.ts.enabled}")
private boolean ttlTaskExecutionEnabled;
@Scheduled(initialDelayString = "${sql.ttl.ts.execution_interval_ms}", fixedDelayString = "${sql.ttl.ts.execution_interval_ms}")
public void cleanUp() {
if (ttlTaskExecutionEnabled) {
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
doCleanUp(conn);
} catch (SQLException e) {
log.error("SQLException occurred during TTL task execution ", e);
}
}
}
}

2
application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java → application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java

@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.thingsboard.server.service.ttl; package org.thingsboard.server.service.ttl.timeseries;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;

2
application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java → application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java

@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.thingsboard.server.service.ttl; package org.thingsboard.server.service.ttl.timeseries;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;

3
application/src/main/resources/logback.xml

@ -31,8 +31,11 @@
<!-- <logger name="org.thingsboard.server.service.queue" level="TRACE" />--> <!-- <logger name="org.thingsboard.server.service.queue" level="TRACE" />-->
<!-- <logger name="org.thingsboard.server.service.transport" level="TRACE" />--> <!-- <logger name="org.thingsboard.server.service.transport" level="TRACE" />-->
<logger name="com.microsoft.azure.servicebus.primitives.CoreMessageReceiver" level="OFF" />
<root level="INFO"> <root level="INFO">
<appender-ref ref="STDOUT"/> <appender-ref ref="STDOUT"/>
</root> </root>
</configuration> </configuration>

107
application/src/main/resources/thingsboard.yml

@ -181,31 +181,37 @@ cassandra:
# SQL configuration parameters # SQL configuration parameters
sql: sql:
# Specify batch size for persisting attribute updates # Specify batch size for persisting attribute updates
attributes: attributes:
batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}" batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}"
batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}" batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}"
stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}" stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}"
ts: ts:
batch_size: "${SQL_TS_BATCH_SIZE:10000}" batch_size: "${SQL_TS_BATCH_SIZE:10000}"
batch_max_delay: "${SQL_TS_BATCH_MAX_DELAY_MS:100}" batch_max_delay: "${SQL_TS_BATCH_MAX_DELAY_MS:100}"
stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}" stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}"
ts_latest: ts_latest:
batch_size: "${SQL_TS_LATEST_BATCH_SIZE:10000}" batch_size: "${SQL_TS_LATEST_BATCH_SIZE:10000}"
batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:100}" batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:100}"
stats_print_interval_ms: "${SQL_TS_LATEST_BATCH_STATS_PRINT_MS:10000}" stats_print_interval_ms: "${SQL_TS_LATEST_BATCH_STATS_PRINT_MS:10000}"
# Specify whether to remove null characters from strValue of attributes and timeseries before insert # Specify whether to remove null characters from strValue of attributes and timeseries before insert
remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}" remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}"
postgres: postgres:
# Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE. # Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE.
ts_key_value_partitioning: "${SQL_POSTGRES_TS_KV_PARTITIONING:MONTHS}" ts_key_value_partitioning: "${SQL_POSTGRES_TS_KV_PARTITIONING:MONTHS}"
timescale: timescale:
# Specify Interval size for new data chunks storage. # Specify Interval size for new data chunks storage.
chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}" chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}"
ttl: ttl:
enabled: "${SQL_TTL_ENABLED:true}" ts:
execution_interval_ms: "${SQL_TTL_EXECUTION_INTERVAL:86400000}" # Number of miliseconds enabled: "${SQL_TTL_TS_ENABLED:true}"
ts_key_value_ttl: "${SQL_TTL_TS_KEY_VALUE_TTL:0}" # Number of seconds execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day
ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds
events:
enabled: "${SQL_TTL_EVENTS_ENABLED:true}"
execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day
events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" # Number of seconds
debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" # Number of seconds. The current value corresponds to one week
# Actor system parameters # Actor system parameters
actors: actors:
@ -410,12 +416,13 @@ audit-log:
password: "${AUDIT_LOG_SINK_PASSWORD:}" password: "${AUDIT_LOG_SINK_PASSWORD:}"
state: state:
defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:10}" # Should be greater then transport.sessions.report_timeout
defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:10}" defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:600}"
defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:60}"
persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}"
js: js:
evaluator: "${JS_EVALUATOR:remote}" # local/remote evaluator: "${JS_EVALUATOR:local}" # local/remote
# Built-in JVM JavaScript environment properties # Built-in JVM JavaScript environment properties
local: local:
# Use Sandboxed (secured) JVM JavaScript environment # Use Sandboxed (secured) JVM JavaScript environment
@ -543,14 +550,25 @@ queue:
pubsub: pubsub:
project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}"
service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}"
ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again
max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes
max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}"
queue-properties:
rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}"
core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}"
transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}"
notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}"
js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}"
service_bus: service_bus:
namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}"
sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}"
sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}"
max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}"
queue-properties:
rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}"
core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}"
transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}"
notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}"
js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}"
rabbitmq: rabbitmq:
exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}"
host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}"
@ -561,6 +579,12 @@ queue:
automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}"
connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}"
handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}"
queue-properties:
rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}"
core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}"
transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}"
notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}"
js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}"
partitions: partitions:
hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}"
virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}"
@ -578,7 +602,7 @@ queue:
partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}"
pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}"
stats: stats:
enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}"
print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}"
js: js:
# JS Eval request topic # JS Eval request topic
@ -607,11 +631,11 @@ queue:
partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}"
pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}"
submit-strategy: submit-strategy:
type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL
# For BATCH only # For BATCH only
batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch
processing-strategy: processing-strategy:
type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:SKIP_ALL_FAILURES}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT
# For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT
retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited
failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages;
@ -619,10 +643,10 @@ queue:
- name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}"
topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}"
poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}"
partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" partitions: "${TB_QUEUE_RE_HP_PARTITIONS:10}"
pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}"
submit-strategy: submit-strategy:
type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL
# For BATCH only # For BATCH only
batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch
processing-strategy: processing-strategy:
@ -631,6 +655,21 @@ queue:
retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited
failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages;
pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries;
- name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}"
topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}"
poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}"
partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:10}"
pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:60000}"
submit-strategy:
type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL
# For BATCH only
batch-size: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch
processing-strategy:
type: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT
# For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT
retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited
failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages;
pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries;
transport: transport:
# For high priority notifications that require minimum latency and processing time # For high priority notifications that require minimum latency and processing time
notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}"

128
application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java

@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.page.TextPageData;
import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.page.TextPageLink;
import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.service.stats.DefaultRuleEngineStatisticsService;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
@ -71,7 +72,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
public void afterTest() throws Exception { public void afterTest() throws Exception {
loginSysAdmin(); loginSysAdmin();
doDelete("/api/tenant/"+savedTenant.getId().getId().toString()) doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
.andExpect(status().isOk()); .andExpect(status().isOk());
} }
@ -111,26 +112,27 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
@Test @Test
public void testFindAssetTypesByTenantId() throws Exception { public void testFindAssetTypesByTenantId() throws Exception {
List<Asset> assets = new ArrayList<>(); List<Asset> assets = new ArrayList<>();
for (int i=0;i<3;i++) { for (int i = 0; i < 3; i++) {
Asset asset = new Asset(); Asset asset = new Asset();
asset.setName("My asset B"+i); asset.setName("My asset B" + i);
asset.setType("typeB"); asset.setType("typeB");
assets.add(doPost("/api/asset", asset, Asset.class)); assets.add(doPost("/api/asset", asset, Asset.class));
} }
for (int i=0;i<7;i++) { for (int i = 0; i < 7; i++) {
Asset asset = new Asset(); Asset asset = new Asset();
asset.setName("My asset C"+i); asset.setName("My asset C" + i);
asset.setType("typeC"); asset.setType("typeC");
assets.add(doPost("/api/asset", asset, Asset.class)); assets.add(doPost("/api/asset", asset, Asset.class));
} }
for (int i=0;i<9;i++) { for (int i = 0; i < 9; i++) {
Asset asset = new Asset(); Asset asset = new Asset();
asset.setName("My asset A"+i); asset.setName("My asset A" + i);
asset.setType("typeA"); asset.setType("typeA");
assets.add(doPost("/api/asset", asset, Asset.class)); assets.add(doPost("/api/asset", asset, Asset.class));
} }
List<EntitySubtype> assetTypes = doGetTyped("/api/asset/types", List<EntitySubtype> assetTypes = doGetTyped("/api/asset/types",
new TypeReference<List<EntitySubtype>>(){}); new TypeReference<List<EntitySubtype>>() {
});
Assert.assertNotNull(assetTypes); Assert.assertNotNull(assetTypes);
Assert.assertEquals(3, assetTypes.size()); Assert.assertEquals(3, assetTypes.size());
@ -146,10 +148,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
asset.setType("default"); asset.setType("default");
Asset savedAsset = doPost("/api/asset", asset, Asset.class); Asset savedAsset = doPost("/api/asset", asset, Asset.class);
doDelete("/api/asset/"+savedAsset.getId().getId().toString()) doDelete("/api/asset/" + savedAsset.getId().getId().toString())
.andExpect(status().isOk()); .andExpect(status().isOk());
doGet("/api/asset/"+savedAsset.getId().getId().toString()) doGet("/api/asset/" + savedAsset.getId().getId().toString())
.andExpect(status().isNotFound()); .andExpect(status().isNotFound());
} }
@ -244,16 +246,16 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
loginSysAdmin(); loginSysAdmin();
doDelete("/api/tenant/"+savedTenant2.getId().getId().toString()) doDelete("/api/tenant/" + savedTenant2.getId().getId().toString())
.andExpect(status().isOk()); .andExpect(status().isOk());
} }
@Test @Test
public void testFindTenantAssets() throws Exception { public void testFindTenantAssets() throws Exception {
List<Asset> assets = new ArrayList<>(); List<Asset> assets = new ArrayList<>();
for (int i=0;i<178;i++) { for (int i = 0; i < 178; i++) {
Asset asset = new Asset(); Asset asset = new Asset();
asset.setName("Asset"+i); asset.setName("Asset" + i);
asset.setType("default"); asset.setType("default");
assets.add(doPost("/api/asset", asset, Asset.class)); assets.add(doPost("/api/asset", asset, Asset.class));
} }
@ -262,13 +264,16 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
TextPageData<Asset> pageData = null; TextPageData<Asset> pageData = null;
do { do {
pageData = doGetTypedWithPageLink("/api/tenant/assets?", pageData = doGetTypedWithPageLink("/api/tenant/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink); new TypeReference<TextPageData<Asset>>() {
}, pageLink);
loadedAssets.addAll(pageData.getData()); loadedAssets.addAll(pageData.getData());
if (pageData.hasNext()) { if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink(); pageLink = pageData.getNextPageLink();
} }
} while (pageData.hasNext()); } while (pageData.hasNext());
loadedAssets.removeIf(asset -> asset.getType().equals(DefaultRuleEngineStatisticsService.TB_SERVICE_QUEUE));
Collections.sort(assets, idComparator); Collections.sort(assets, idComparator);
Collections.sort(loadedAssets, idComparator); Collections.sort(loadedAssets, idComparator);
@ -279,10 +284,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
public void testFindTenantAssetsByName() throws Exception { public void testFindTenantAssetsByName() throws Exception {
String title1 = "Asset title 1"; String title1 = "Asset title 1";
List<Asset> assetsTitle1 = new ArrayList<>(); List<Asset> assetsTitle1 = new ArrayList<>();
for (int i=0;i<143;i++) { for (int i = 0; i < 143; i++) {
Asset asset = new Asset(); Asset asset = new Asset();
String suffix = RandomStringUtils.randomAlphanumeric(15); String suffix = RandomStringUtils.randomAlphanumeric(15);
String name = title1+suffix; String name = title1 + suffix;
name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase();
asset.setName(name); asset.setName(name);
asset.setType("default"); asset.setType("default");
@ -290,10 +295,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
} }
String title2 = "Asset title 2"; String title2 = "Asset title 2";
List<Asset> assetsTitle2 = new ArrayList<>(); List<Asset> assetsTitle2 = new ArrayList<>();
for (int i=0;i<75;i++) { for (int i = 0; i < 75; i++) {
Asset asset = new Asset(); Asset asset = new Asset();
String suffix = RandomStringUtils.randomAlphanumeric(15); String suffix = RandomStringUtils.randomAlphanumeric(15);
String name = title2+suffix; String name = title2 + suffix;
name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase();
asset.setName(name); asset.setName(name);
asset.setType("default"); asset.setType("default");
@ -305,7 +310,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
TextPageData<Asset> pageData = null; TextPageData<Asset> pageData = null;
do { do {
pageData = doGetTypedWithPageLink("/api/tenant/assets?", pageData = doGetTypedWithPageLink("/api/tenant/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink); new TypeReference<TextPageData<Asset>>() {
}, pageLink);
loadedAssetsTitle1.addAll(pageData.getData()); loadedAssetsTitle1.addAll(pageData.getData());
if (pageData.hasNext()) { if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink(); pageLink = pageData.getNextPageLink();
@ -321,7 +327,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
pageLink = new TextPageLink(4, title2); pageLink = new TextPageLink(4, title2);
do { do {
pageData = doGetTypedWithPageLink("/api/tenant/assets?", pageData = doGetTypedWithPageLink("/api/tenant/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink); new TypeReference<TextPageData<Asset>>() {
}, pageLink);
loadedAssetsTitle2.addAll(pageData.getData()); loadedAssetsTitle2.addAll(pageData.getData());
if (pageData.hasNext()) { if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink(); pageLink = pageData.getNextPageLink();
@ -334,24 +341,26 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
Assert.assertEquals(assetsTitle2, loadedAssetsTitle2); Assert.assertEquals(assetsTitle2, loadedAssetsTitle2);
for (Asset asset : loadedAssetsTitle1) { for (Asset asset : loadedAssetsTitle1) {
doDelete("/api/asset/"+asset.getId().getId().toString()) doDelete("/api/asset/" + asset.getId().getId().toString())
.andExpect(status().isOk()); .andExpect(status().isOk());
} }
pageLink = new TextPageLink(4, title1); pageLink = new TextPageLink(4, title1);
pageData = doGetTypedWithPageLink("/api/tenant/assets?", pageData = doGetTypedWithPageLink("/api/tenant/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink); new TypeReference<TextPageData<Asset>>() {
}, pageLink);
Assert.assertFalse(pageData.hasNext()); Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size()); Assert.assertEquals(0, pageData.getData().size());
for (Asset asset : loadedAssetsTitle2) { for (Asset asset : loadedAssetsTitle2) {
doDelete("/api/asset/"+asset.getId().getId().toString()) doDelete("/api/asset/" + asset.getId().getId().toString())
.andExpect(status().isOk()); .andExpect(status().isOk());
} }
pageLink = new TextPageLink(4, title2); pageLink = new TextPageLink(4, title2);
pageData = doGetTypedWithPageLink("/api/tenant/assets?", pageData = doGetTypedWithPageLink("/api/tenant/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink); new TypeReference<TextPageData<Asset>>() {
}, pageLink);
Assert.assertFalse(pageData.hasNext()); Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size()); Assert.assertEquals(0, pageData.getData().size());
} }
@ -361,10 +370,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
String title1 = "Asset title 1"; String title1 = "Asset title 1";
String type1 = "typeA"; String type1 = "typeA";
List<Asset> assetsType1 = new ArrayList<>(); List<Asset> assetsType1 = new ArrayList<>();
for (int i=0;i<143;i++) { for (int i = 0; i < 143; i++) {
Asset asset = new Asset(); Asset asset = new Asset();
String suffix = RandomStringUtils.randomAlphanumeric(15); String suffix = RandomStringUtils.randomAlphanumeric(15);
String name = title1+suffix; String name = title1 + suffix;
name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase();
asset.setName(name); asset.setName(name);
asset.setType(type1); asset.setType(type1);
@ -373,10 +382,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
String title2 = "Asset title 2"; String title2 = "Asset title 2";
String type2 = "typeB"; String type2 = "typeB";
List<Asset> assetsType2 = new ArrayList<>(); List<Asset> assetsType2 = new ArrayList<>();
for (int i=0;i<75;i++) { for (int i = 0; i < 75; i++) {
Asset asset = new Asset(); Asset asset = new Asset();
String suffix = RandomStringUtils.randomAlphanumeric(15); String suffix = RandomStringUtils.randomAlphanumeric(15);
String name = title2+suffix; String name = title2 + suffix;
name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase();
asset.setName(name); asset.setName(name);
asset.setType(type2); asset.setType(type2);
@ -388,7 +397,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
TextPageData<Asset> pageData = null; TextPageData<Asset> pageData = null;
do { do {
pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&",
new TypeReference<TextPageData<Asset>>(){}, pageLink, type1); new TypeReference<TextPageData<Asset>>() {
}, pageLink, type1);
loadedAssetsType1.addAll(pageData.getData()); loadedAssetsType1.addAll(pageData.getData());
if (pageData.hasNext()) { if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink(); pageLink = pageData.getNextPageLink();
@ -404,7 +414,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
pageLink = new TextPageLink(4); pageLink = new TextPageLink(4);
do { do {
pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&",
new TypeReference<TextPageData<Asset>>(){}, pageLink, type2); new TypeReference<TextPageData<Asset>>() {
}, pageLink, type2);
loadedAssetsType2.addAll(pageData.getData()); loadedAssetsType2.addAll(pageData.getData());
if (pageData.hasNext()) { if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink(); pageLink = pageData.getNextPageLink();
@ -417,24 +428,26 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
Assert.assertEquals(assetsType2, loadedAssetsType2); Assert.assertEquals(assetsType2, loadedAssetsType2);
for (Asset asset : loadedAssetsType1) { for (Asset asset : loadedAssetsType1) {
doDelete("/api/asset/"+asset.getId().getId().toString()) doDelete("/api/asset/" + asset.getId().getId().toString())
.andExpect(status().isOk()); .andExpect(status().isOk());
} }
pageLink = new TextPageLink(4); pageLink = new TextPageLink(4);
pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&",
new TypeReference<TextPageData<Asset>>(){}, pageLink, type1); new TypeReference<TextPageData<Asset>>() {
}, pageLink, type1);
Assert.assertFalse(pageData.hasNext()); Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size()); Assert.assertEquals(0, pageData.getData().size());
for (Asset asset : loadedAssetsType2) { for (Asset asset : loadedAssetsType2) {
doDelete("/api/asset/"+asset.getId().getId().toString()) doDelete("/api/asset/" + asset.getId().getId().toString())
.andExpect(status().isOk()); .andExpect(status().isOk());
} }
pageLink = new TextPageLink(4); pageLink = new TextPageLink(4);
pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&",
new TypeReference<TextPageData<Asset>>(){}, pageLink, type2); new TypeReference<TextPageData<Asset>>() {
}, pageLink, type2);
Assert.assertFalse(pageData.hasNext()); Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size()); Assert.assertEquals(0, pageData.getData().size());
} }
@ -447,9 +460,9 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
CustomerId customerId = customer.getId(); CustomerId customerId = customer.getId();
List<Asset> assets = new ArrayList<>(); List<Asset> assets = new ArrayList<>();
for (int i=0;i<128;i++) { for (int i = 0; i < 128; i++) {
Asset asset = new Asset(); Asset asset = new Asset();
asset.setName("Asset"+i); asset.setName("Asset" + i);
asset.setType("default"); asset.setType("default");
asset = doPost("/api/asset", asset, Asset.class); asset = doPost("/api/asset", asset, Asset.class);
assets.add(doPost("/api/customer/" + customerId.getId().toString() assets.add(doPost("/api/customer/" + customerId.getId().toString()
@ -461,7 +474,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
TextPageData<Asset> pageData = null; TextPageData<Asset> pageData = null;
do { do {
pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink); new TypeReference<TextPageData<Asset>>() {
}, pageLink);
loadedAssets.addAll(pageData.getData()); loadedAssets.addAll(pageData.getData());
if (pageData.hasNext()) { if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink(); pageLink = pageData.getNextPageLink();
@ -483,10 +497,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
String title1 = "Asset title 1"; String title1 = "Asset title 1";
List<Asset> assetsTitle1 = new ArrayList<>(); List<Asset> assetsTitle1 = new ArrayList<>();
for (int i=0;i<125;i++) { for (int i = 0; i < 125; i++) {
Asset asset = new Asset(); Asset asset = new Asset();
String suffix = RandomStringUtils.randomAlphanumeric(15); String suffix = RandomStringUtils.randomAlphanumeric(15);
String name = title1+suffix; String name = title1 + suffix;
name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase();
asset.setName(name); asset.setName(name);
asset.setType("default"); asset.setType("default");
@ -496,10 +510,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
} }
String title2 = "Asset title 2"; String title2 = "Asset title 2";
List<Asset> assetsTitle2 = new ArrayList<>(); List<Asset> assetsTitle2 = new ArrayList<>();
for (int i=0;i<143;i++) { for (int i = 0; i < 143; i++) {
Asset asset = new Asset(); Asset asset = new Asset();
String suffix = RandomStringUtils.randomAlphanumeric(15); String suffix = RandomStringUtils.randomAlphanumeric(15);
String name = title2+suffix; String name = title2 + suffix;
name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase();
asset.setName(name); asset.setName(name);
asset.setType("default"); asset.setType("default");
@ -513,7 +527,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
TextPageData<Asset> pageData = null; TextPageData<Asset> pageData = null;
do { do {
pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink); new TypeReference<TextPageData<Asset>>() {
}, pageLink);
loadedAssetsTitle1.addAll(pageData.getData()); loadedAssetsTitle1.addAll(pageData.getData());
if (pageData.hasNext()) { if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink(); pageLink = pageData.getNextPageLink();
@ -529,7 +544,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
pageLink = new TextPageLink(4, title2); pageLink = new TextPageLink(4, title2);
do { do {
pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink); new TypeReference<TextPageData<Asset>>() {
}, pageLink);
loadedAssetsTitle2.addAll(pageData.getData()); loadedAssetsTitle2.addAll(pageData.getData());
if (pageData.hasNext()) { if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink(); pageLink = pageData.getNextPageLink();
@ -548,7 +564,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
pageLink = new TextPageLink(4, title1); pageLink = new TextPageLink(4, title1);
pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink); new TypeReference<TextPageData<Asset>>() {
}, pageLink);
Assert.assertFalse(pageData.hasNext()); Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size()); Assert.assertEquals(0, pageData.getData().size());
@ -559,7 +576,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
pageLink = new TextPageLink(4, title2); pageLink = new TextPageLink(4, title2);
pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink); new TypeReference<TextPageData<Asset>>() {
}, pageLink);
Assert.assertFalse(pageData.hasNext()); Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size()); Assert.assertEquals(0, pageData.getData().size());
} }
@ -574,10 +592,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
String title1 = "Asset title 1"; String title1 = "Asset title 1";
String type1 = "typeC"; String type1 = "typeC";
List<Asset> assetsType1 = new ArrayList<>(); List<Asset> assetsType1 = new ArrayList<>();
for (int i=0;i<125;i++) { for (int i = 0; i < 125; i++) {
Asset asset = new Asset(); Asset asset = new Asset();
String suffix = RandomStringUtils.randomAlphanumeric(15); String suffix = RandomStringUtils.randomAlphanumeric(15);
String name = title1+suffix; String name = title1 + suffix;
name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase();
asset.setName(name); asset.setName(name);
asset.setType(type1); asset.setType(type1);
@ -588,10 +606,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
String title2 = "Asset title 2"; String title2 = "Asset title 2";
String type2 = "typeD"; String type2 = "typeD";
List<Asset> assetsType2 = new ArrayList<>(); List<Asset> assetsType2 = new ArrayList<>();
for (int i=0;i<143;i++) { for (int i = 0; i < 143; i++) {
Asset asset = new Asset(); Asset asset = new Asset();
String suffix = RandomStringUtils.randomAlphanumeric(15); String suffix = RandomStringUtils.randomAlphanumeric(15);
String name = title2+suffix; String name = title2 + suffix;
name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase();
asset.setName(name); asset.setName(name);
asset.setType(type2); asset.setType(type2);
@ -605,7 +623,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
TextPageData<Asset> pageData = null; TextPageData<Asset> pageData = null;
do { do {
pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&",
new TypeReference<TextPageData<Asset>>(){}, pageLink, type1); new TypeReference<TextPageData<Asset>>() {
}, pageLink, type1);
loadedAssetsType1.addAll(pageData.getData()); loadedAssetsType1.addAll(pageData.getData());
if (pageData.hasNext()) { if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink(); pageLink = pageData.getNextPageLink();
@ -621,7 +640,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
pageLink = new TextPageLink(4); pageLink = new TextPageLink(4);
do { do {
pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&",
new TypeReference<TextPageData<Asset>>(){}, pageLink, type2); new TypeReference<TextPageData<Asset>>() {
}, pageLink, type2);
loadedAssetsType2.addAll(pageData.getData()); loadedAssetsType2.addAll(pageData.getData());
if (pageData.hasNext()) { if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink(); pageLink = pageData.getNextPageLink();
@ -640,7 +660,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
pageLink = new TextPageLink(4); pageLink = new TextPageLink(4);
pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&",
new TypeReference<TextPageData<Asset>>(){}, pageLink, type1); new TypeReference<TextPageData<Asset>>() {
}, pageLink, type1);
Assert.assertFalse(pageData.hasNext()); Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size()); Assert.assertEquals(0, pageData.getData().size());
@ -651,7 +672,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
pageLink = new TextPageLink(4); pageLink = new TextPageLink(4);
pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&",
new TypeReference<TextPageData<Asset>>(){}, pageLink, type2); new TypeReference<TextPageData<Asset>>() {
}, pageLink, type2);
Assert.assertFalse(pageData.hasNext()); Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size()); Assert.assertEquals(0, pageData.getData().size());
} }

6
application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java

@ -426,7 +426,7 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes
message.setPayload(strKvs.getBytes()); message.setPayload(strKvs.getBytes());
client.publish("v1/devices/me/telemetry", message); client.publish("v1/devices/me/telemetry", message);
Thread.sleep(1000); Thread.sleep(1000);
// client.disconnect(); client.disconnect();
} }
private void awaitConnected(MqttAsyncClient client, long ms) throws InterruptedException { private void awaitConnected(MqttAsyncClient client, long ms) throws InterruptedException {
@ -463,13 +463,13 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes
MqttConnectOptions options = new MqttConnectOptions(); MqttConnectOptions options = new MqttConnectOptions();
options.setUserName(accessToken); options.setUserName(accessToken);
client.connect(options); client.connect(options);
Thread.sleep(3000); awaitConnected(client, TimeUnit.SECONDS.toMillis(30));
MqttMessage message = new MqttMessage(); MqttMessage message = new MqttMessage();
message.setPayload((stringKV).getBytes()); message.setPayload((stringKV).getBytes());
client.publish("v1/devices/me/attributes", message); client.publish("v1/devices/me/attributes", message);
Thread.sleep(1000); Thread.sleep(1000);
client.disconnect();
return new HashSet<>(doGetAsync("/api/plugins/telemetry/DEVICE/" + viewDeviceId + "/keys/attributes", List.class)); return new HashSet<>(doGetAsync("/api/plugins/telemetry/DEVICE/" + viewDeviceId + "/keys/attributes", List.class));
} }

7
application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java

@ -16,10 +16,12 @@
package org.thingsboard.server.controller; package org.thingsboard.server.controller;
import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet;
import org.junit.BeforeClass;
import org.junit.ClassRule; import org.junit.ClassRule;
import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.thingsboard.server.dao.CustomCassandraCQLUnit; import org.thingsboard.server.dao.CustomCassandraCQLUnit;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import java.util.Arrays; import java.util.Arrays;
@ -37,4 +39,9 @@ public class ControllerNoSqlTestSuite {
new ClassPathCQLDataSet("cassandra/system-data.cql", false, false), new ClassPathCQLDataSet("cassandra/system-data.cql", false, false),
new ClassPathCQLDataSet("cassandra/system-test.cql", false, false)), new ClassPathCQLDataSet("cassandra/system-test.cql", false, false)),
"cassandra-test.yaml", 30000l); "cassandra-test.yaml", 30000l);
@BeforeClass
public static void cleanupInMemStorage(){
InMemoryStorage.getInstance().cleanup();
}
} }

7
application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java

@ -15,10 +15,12 @@
*/ */
package org.thingsboard.server.controller; package org.thingsboard.server.controller;
import org.junit.BeforeClass;
import org.junit.ClassRule; import org.junit.ClassRule;
import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.thingsboard.server.dao.CustomSqlUnit; import org.thingsboard.server.dao.CustomSqlUnit;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import java.util.Arrays; import java.util.Arrays;
@ -33,4 +35,9 @@ public class ControllerSqlTestSuite {
Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql"), Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql"),
"sql/drop-all-tables.sql", "sql/drop-all-tables.sql",
"sql-test.properties"); "sql-test.properties");
@BeforeClass
public static void cleanupInMemStorage(){
InMemoryStorage.getInstance().cleanup();
}
} }

7
application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java

@ -16,10 +16,12 @@
package org.thingsboard.server.mqtt; package org.thingsboard.server.mqtt;
import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet;
import org.junit.BeforeClass;
import org.junit.ClassRule; import org.junit.ClassRule;
import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.thingsboard.server.dao.CustomCassandraCQLUnit; import org.thingsboard.server.dao.CustomCassandraCQLUnit;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import java.util.Arrays; import java.util.Arrays;
@ -36,4 +38,9 @@ public class MqttNoSqlTestSuite {
new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false), new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false),
new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)), new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)),
"cassandra-test.yaml", 30000l); "cassandra-test.yaml", 30000l);
@BeforeClass
public static void cleanupInMemStorage(){
InMemoryStorage.getInstance().cleanup();
}
} }

7
application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java

@ -15,10 +15,12 @@
*/ */
package org.thingsboard.server.mqtt; package org.thingsboard.server.mqtt;
import org.junit.BeforeClass;
import org.junit.ClassRule; import org.junit.ClassRule;
import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.thingsboard.server.dao.CustomSqlUnit; import org.thingsboard.server.dao.CustomSqlUnit;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import java.util.Arrays; import java.util.Arrays;
@ -32,4 +34,9 @@ public class MqttSqlTestSuite {
Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"),
"sql/drop-all-tables.sql", "sql/drop-all-tables.sql",
"sql-test.properties"); "sql-test.properties");
@BeforeClass
public static void cleanupInMemStorage(){
InMemoryStorage.getInstance().cleanup();
}
} }

19
application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java

@ -37,6 +37,7 @@ import org.thingsboard.server.service.security.AccessValidator;
import java.util.Arrays; import java.util.Arrays;
import java.util.concurrent.CountDownLatch; import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotNull;
@ -55,6 +56,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC
private User tenantAdmin; private User tenantAdmin;
private Long asyncContextTimeoutToUseRpcPlugin; private Long asyncContextTimeoutToUseRpcPlugin;
private static final AtomicInteger atomicInteger = new AtomicInteger(2);
@Before @Before
public void beforeTest() throws Exception { public void beforeTest() throws Exception {
@ -70,7 +73,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC
tenantAdmin = new User(); tenantAdmin = new User();
tenantAdmin.setAuthority(Authority.TENANT_ADMIN); tenantAdmin.setAuthority(Authority.TENANT_ADMIN);
tenantAdmin.setTenantId(savedTenant.getId()); tenantAdmin.setTenantId(savedTenant.getId());
tenantAdmin.setEmail("tenant2@thingsboard.org"); tenantAdmin.setEmail("tenant" + atomicInteger.getAndIncrement() + "@thingsboard.org");
tenantAdmin.setFirstName("Joe"); tenantAdmin.setFirstName("Joe");
tenantAdmin.setLastName("Downs"); tenantAdmin.setLastName("Downs");
@ -109,6 +112,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC
client.subscribe("v1/devices/me/rpc/request/+", MqttQoS.AT_MOST_ONCE.value()); client.subscribe("v1/devices/me/rpc/request/+", MqttQoS.AT_MOST_ONCE.value());
Thread.sleep(2000);
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}";
String deviceId = savedDevice.getId().getId().toString(); String deviceId = savedDevice.getId().getId().toString();
String result = doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isOk()); String result = doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isOk());
@ -128,7 +133,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC
String accessToken = deviceCredentials.getCredentialsId(); String accessToken = deviceCredentials.getCredentialsId();
assertNotNull(accessToken); assertNotNull(accessToken);
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"timeout\": 6000}"; String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"24\",\"value\": 1},\"timeout\": 6000}";
String deviceId = savedDevice.getId().getId().toString(); String deviceId = savedDevice.getId().getId().toString();
doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(), doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(),
@ -137,7 +142,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC
@Test @Test
public void testServerMqttOneWayRpcDeviceDoesNotExist() throws Exception { public void testServerMqttOneWayRpcDeviceDoesNotExist() throws Exception {
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"25\",\"value\": 1}}";
String nonExistentDeviceId = UUIDs.timeBased().toString(); String nonExistentDeviceId = UUIDs.timeBased().toString();
String result = doPostAsync("/api/plugins/rpc/oneway/" + nonExistentDeviceId, setGpioRequest, String.class, String result = doPostAsync("/api/plugins/rpc/oneway/" + nonExistentDeviceId, setGpioRequest, String.class,
@ -165,7 +170,9 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC
client.subscribe("v1/devices/me/rpc/request/+", 1); client.subscribe("v1/devices/me/rpc/request/+", 1);
client.setCallback(new TestMqttCallback(client, new CountDownLatch(1))); client.setCallback(new TestMqttCallback(client, new CountDownLatch(1)));
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; Thread.sleep(2000);
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"26\",\"value\": 1}}";
String deviceId = savedDevice.getId().getId().toString(); String deviceId = savedDevice.getId().getId().toString();
String result = doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isOk()); String result = doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isOk());
@ -183,7 +190,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC
String accessToken = deviceCredentials.getCredentialsId(); String accessToken = deviceCredentials.getCredentialsId();
assertNotNull(accessToken); assertNotNull(accessToken);
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"timeout\": 6000}"; String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"27\",\"value\": 1},\"timeout\": 6000}";
String deviceId = savedDevice.getId().getId().toString(); String deviceId = savedDevice.getId().getId().toString();
doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(), doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(),
@ -192,7 +199,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC
@Test @Test
public void testServerMqttTwoWayRpcDeviceDoesNotExist() throws Exception { public void testServerMqttTwoWayRpcDeviceDoesNotExist() throws Exception {
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"28\",\"value\": 1}}";
String nonExistentDeviceId = UUIDs.timeBased().toString(); String nonExistentDeviceId = UUIDs.timeBased().toString();
String result = doPostAsync("/api/plugins/rpc/twoway/" + nonExistentDeviceId, setGpioRequest, String.class, String result = doPostAsync("/api/plugins/rpc/twoway/" + nonExistentDeviceId, setGpioRequest, String.class,

11
application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java

@ -79,7 +79,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr
String deviceId = savedDevice.getId().getId().toString(); String deviceId = savedDevice.getId().getId().toString();
Thread.sleep(1000); Thread.sleep(2000);
List<String> actualKeys = doGetAsync("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", List.class); List<String> actualKeys = doGetAsync("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", List.class);
Set<String> actualKeySet = new HashSet<>(actualKeys); Set<String> actualKeySet = new HashSet<>(actualKeys);
@ -97,7 +97,8 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr
assertEquals("4", values.get("key4").get(0).get("value")); assertEquals("4", values.get("key4").get(0).get("value"));
} }
@Test
// @Test - Unstable
public void testMqttQoSLevel() throws Exception { public void testMqttQoSLevel() throws Exception {
String clientId = MqttAsyncClient.generateClientId(); String clientId = MqttAsyncClient.generateClientId();
MqttAsyncClient client = new MqttAsyncClient(MQTT_URL, clientId); MqttAsyncClient client = new MqttAsyncClient(MQTT_URL, clientId);
@ -107,13 +108,13 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr
CountDownLatch latch = new CountDownLatch(1); CountDownLatch latch = new CountDownLatch(1);
TestMqttCallback callback = new TestMqttCallback(client, latch); TestMqttCallback callback = new TestMqttCallback(client, latch);
client.setCallback(callback); client.setCallback(callback);
client.connect(options).waitForCompletion(3000); client.connect(options).waitForCompletion(5000);
client.subscribe("v1/devices/me/attributes", MqttQoS.AT_MOST_ONCE.value()); client.subscribe("v1/devices/me/attributes", MqttQoS.AT_MOST_ONCE.value());
String payload = "{\"key\":\"value\"}"; String payload = "{\"key\":\"uniqueValue\"}";
// TODO 3.1: we need to acknowledge subscription only after it is processed by device actor and not when the message is pushed to queue. // TODO 3.1: we need to acknowledge subscription only after it is processed by device actor and not when the message is pushed to queue.
// MqttClient -> SUB REQUEST -> Transport -> Kafka -> Device Actor (subscribed) // MqttClient -> SUB REQUEST -> Transport -> Kafka -> Device Actor (subscribed)
// MqttClient <- SUB_ACK <- Transport // MqttClient <- SUB_ACK <- Transport
Thread.sleep(1000); Thread.sleep(5000);
doPostAsync("/api/plugins/telemetry/" + savedDevice.getId() + "/SHARED_SCOPE", payload, String.class, status().isOk()); doPostAsync("/api/plugins/telemetry/" + savedDevice.getId() + "/SHARED_SCOPE", payload, String.class, status().isOk());
latch.await(10, TimeUnit.SECONDS); latch.await(10, TimeUnit.SECONDS);
assertEquals(payload, callback.getPayload()); assertEquals(payload, callback.getPayload());

7
application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java

@ -16,11 +16,13 @@
package org.thingsboard.server.rules; package org.thingsboard.server.rules;
import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet;
import org.junit.BeforeClass;
import org.junit.ClassRule; import org.junit.ClassRule;
import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.thingsboard.server.dao.CustomCassandraCQLUnit; import org.thingsboard.server.dao.CustomCassandraCQLUnit;
import org.thingsboard.server.dao.CustomSqlUnit; import org.thingsboard.server.dao.CustomSqlUnit;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import java.util.Arrays; import java.util.Arrays;
@ -40,4 +42,9 @@ public class RuleEngineNoSqlTestSuite {
new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)), new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)),
"cassandra-test.yaml", 30000l); "cassandra-test.yaml", 30000l);
@BeforeClass
public static void cleanupInMemStorage(){
InMemoryStorage.getInstance().cleanup();
}
} }

7
application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java

@ -15,10 +15,12 @@
*/ */
package org.thingsboard.server.rules; package org.thingsboard.server.rules;
import org.junit.BeforeClass;
import org.junit.ClassRule; import org.junit.ClassRule;
import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.thingsboard.server.dao.CustomSqlUnit; import org.thingsboard.server.dao.CustomSqlUnit;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import java.util.Arrays; import java.util.Arrays;
@ -33,4 +35,9 @@ public class RuleEngineSqlTestSuite {
Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"),
"sql/drop-all-tables.sql", "sql/drop-all-tables.sql",
"sql-test.properties"); "sql-test.properties");
@BeforeClass
public static void cleanupInMemStorage(){
InMemoryStorage.getInstance().cleanup();
}
} }

4
application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java

@ -152,7 +152,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule
QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null);
// Pushing Message to the system // Pushing Message to the system
actorSystem.tell(qMsg, ActorRef.noSender()); actorSystem.tell(qMsg, ActorRef.noSender());
Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess(); Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess();
TimePageData<Event> eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); TimePageData<Event> eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000);
List<Event> events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); List<Event> events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList());
@ -265,7 +265,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule
// Pushing Message to the system // Pushing Message to the system
actorSystem.tell(qMsg, ActorRef.noSender()); actorSystem.tell(qMsg, ActorRef.noSender());
Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess(); Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess();
TimePageData<Event> eventsPage = getDebugEvents(savedTenant.getId(), rootRuleChain.getFirstRuleNodeId(), 1000); TimePageData<Event> eventsPage = getDebugEvents(savedTenant.getId(), rootRuleChain.getFirstRuleNodeId(), 1000);
List<Event> events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); List<Event> events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList());

21
application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java

@ -17,6 +17,7 @@ package org.thingsboard.server.service.cluster.routing;
import com.datastax.driver.core.utils.UUIDs; import com.datastax.driver.core.utils.UUIDs;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
@ -31,6 +32,8 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; import org.thingsboard.server.queue.discovery.TenantRoutingInfoService;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
@ -41,6 +44,7 @@ import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@Slf4j @Slf4j
@RunWith(MockitoJUnitRunner.class) @RunWith(MockitoJUnitRunner.class)
@ -52,6 +56,7 @@ public class ConsistentHashParitionServiceTest {
private TbServiceInfoProvider discoveryService; private TbServiceInfoProvider discoveryService;
private TenantRoutingInfoService routingInfoService; private TenantRoutingInfoService routingInfoService;
private ApplicationEventPublisher applicationEventPublisher; private ApplicationEventPublisher applicationEventPublisher;
private TbQueueRuleEngineSettings ruleEngineSettings;
private String hashFunctionName = "murmur3_128"; private String hashFunctionName = "murmur3_128";
private Integer virtualNodesSize = 16; private Integer virtualNodesSize = 16;
@ -62,12 +67,15 @@ public class ConsistentHashParitionServiceTest {
discoveryService = mock(TbServiceInfoProvider.class); discoveryService = mock(TbServiceInfoProvider.class);
applicationEventPublisher = mock(ApplicationEventPublisher.class); applicationEventPublisher = mock(ApplicationEventPublisher.class);
routingInfoService = mock(TenantRoutingInfoService.class); routingInfoService = mock(TenantRoutingInfoService.class);
clusterRoutingService = new ConsistentHashPartitionService(discoveryService, routingInfoService, applicationEventPublisher); ruleEngineSettings = mock(TbQueueRuleEngineSettings.class);
clusterRoutingService = new ConsistentHashPartitionService(discoveryService,
routingInfoService,
applicationEventPublisher,
ruleEngineSettings
);
when(ruleEngineSettings.getQueues()).thenReturn(Collections.emptyList());
ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core");
ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 3); ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 3);
ReflectionTestUtils.setField(clusterRoutingService, "ruleEngineTopic", "tb.rule-engine");
ReflectionTestUtils.setField(clusterRoutingService, "ruleEnginePartitions", 100);
ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName);
ReflectionTestUtils.setField(clusterRoutingService, "virtualNodesSize", virtualNodesSize); ReflectionTestUtils.setField(clusterRoutingService, "virtualNodesSize", virtualNodesSize);
TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder()
@ -107,8 +115,9 @@ public class ConsistentHashParitionServiceTest {
List<Map.Entry<Integer, Integer>> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); List<Map.Entry<Integer, Integer>> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList());
long end = System.currentTimeMillis(); long end = System.currentTimeMillis();
double diff = (data.get(data.size() - 1).getValue() - data.get(0).getValue()); double diff = (data.get(data.size() - 1).getValue() - data.get(0).getValue());
System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", (diff / ITERATIONS) * 100.0) + "%)"); double diffPercent = (diff / ITERATIONS) * 100.0;
System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", diffPercent) + "%)");
Assert.assertTrue(diffPercent < 0.5);
for (Map.Entry<Integer, Integer> entry : data) { for (Map.Entry<Integer, Integer> entry : data) {
System.out.println(entry.getKey() + ": " + entry.getValue()); System.out.println(entry.getKey() + ": " + entry.getValue());
} }

66
application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java

@ -0,0 +1,66 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.queue;
import lombok.extern.slf4j.Slf4j;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.mockito.runners.MockitoJUnitRunner;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@Slf4j
@RunWith(MockitoJUnitRunner.class)
public class TbMsgPackProcessingContextTest {
@Test
public void testHighConcurrencyCase() throws InterruptedException {
TbRuleEngineSubmitStrategy strategyMock = mock(TbRuleEngineSubmitStrategy.class);
int msgCount = 1000;
int parallelCount = 5;
ExecutorService executorService = Executors.newFixedThreadPool(parallelCount);
try {
ConcurrentMap<UUID, TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> messages = new ConcurrentHashMap<>();
for (int i = 0; i < msgCount; i++) {
messages.put(UUID.randomUUID(), new TbProtoQueueMsg<>(UUID.randomUUID(), null));
}
when(strategyMock.getPendingMap()).thenReturn(messages);
TbMsgPackProcessingContext context = new TbMsgPackProcessingContext(strategyMock);
for (UUID uuid : messages.keySet()) {
for (int i = 0; i < parallelCount; i++) {
executorService.submit(() -> context.onSuccess(uuid));
}
}
Assert.assertTrue(context.await(10, TimeUnit.SECONDS));
Mockito.verify(strategyMock, Mockito.times(msgCount)).onSuccess(Mockito.any(UUID.class));
} finally {
executorService.shutdownNow();
}
}
}

7
application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java

@ -16,10 +16,12 @@
package org.thingsboard.server.system; package org.thingsboard.server.system;
import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet;
import org.junit.BeforeClass;
import org.junit.ClassRule; import org.junit.ClassRule;
import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.thingsboard.server.dao.CustomCassandraCQLUnit; import org.thingsboard.server.dao.CustomCassandraCQLUnit;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import java.util.Arrays; import java.util.Arrays;
@ -38,4 +40,9 @@ public class SystemNoSqlTestSuite {
new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false), new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false),
new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)), new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)),
"cassandra-test.yaml", 30000l); "cassandra-test.yaml", 30000l);
@BeforeClass
public static void cleanupInMemStorage(){
InMemoryStorage.getInstance().cleanup();
}
} }

7
application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java

@ -15,10 +15,12 @@
*/ */
package org.thingsboard.server.system; package org.thingsboard.server.system;
import org.junit.BeforeClass;
import org.junit.ClassRule; import org.junit.ClassRule;
import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.thingsboard.server.dao.CustomSqlUnit; import org.thingsboard.server.dao.CustomSqlUnit;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import java.util.Arrays; import java.util.Arrays;
@ -35,4 +37,9 @@ public class SystemSqlTestSuite {
"sql/drop-all-tables.sql", "sql/drop-all-tables.sql",
"sql-test.properties"); "sql-test.properties");
@BeforeClass
public static void cleanupInMemStorage(){
InMemoryStorage.getInstance().cleanup();
}
} }

38
common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java

@ -16,27 +16,32 @@
package org.thingsboard.server.queue.azure.servicebus; package org.thingsboard.server.queue.azure.servicebus;
import com.microsoft.azure.servicebus.management.ManagementClient; import com.microsoft.azure.servicebus.management.ManagementClient;
import com.microsoft.azure.servicebus.management.QueueDescription;
import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder;
import com.microsoft.azure.servicebus.primitives.ServiceBusException; import com.microsoft.azure.servicebus.primitives.ServiceBusException;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Component;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
import java.io.IOException; import java.io.IOException;
import java.time.Duration;
import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
@Slf4j @Slf4j
@Component
@ConditionalOnExpression("'${queue.type:null}'=='service-bus'")
public class TbServiceBusAdmin implements TbQueueAdmin { public class TbServiceBusAdmin implements TbQueueAdmin {
private final String MAX_SIZE = "maxSizeInMb";
private final String MESSAGE_TIME_TO_LIVE = "messageTimeToLiveInSec";
private final String LOCK_DURATION = "lockDurationInSec";
private final Map<String, String> queueConfigs;
private final Set<String> queues = ConcurrentHashMap.newKeySet(); private final Set<String> queues = ConcurrentHashMap.newKeySet();
private final ManagementClient client; private final ManagementClient client;
public TbServiceBusAdmin(TbServiceBusSettings serviceBusSettings) { public TbServiceBusAdmin(TbServiceBusSettings serviceBusSettings, Map<String, String> queueConfigs) {
this.queueConfigs = queueConfigs;
ConnectionStringBuilder builder = new ConnectionStringBuilder( ConnectionStringBuilder builder = new ConnectionStringBuilder(
serviceBusSettings.getNamespaceName(), serviceBusSettings.getNamespaceName(),
"queues", "queues",
@ -60,13 +65,34 @@ public class TbServiceBusAdmin implements TbQueueAdmin {
} }
try { try {
client.createQueue(topic); QueueDescription queueDescription = new QueueDescription(topic);
setQueueConfigs(queueDescription);
client.createQueue(queueDescription);
queues.add(topic); queues.add(topic);
} catch (ServiceBusException | InterruptedException e) { } catch (ServiceBusException | InterruptedException e) {
log.error("Failed to create queue: [{}]", topic, e); log.error("Failed to create queue: [{}]", topic, e);
} }
} }
private void setQueueConfigs(QueueDescription queueDescription) {
queueConfigs.forEach((confKey, confValue) -> {
switch (confKey) {
case MAX_SIZE:
queueDescription.setMaxSizeInMB(Long.parseLong(confValue));
break;
case MESSAGE_TIME_TO_LIVE:
queueDescription.setDefaultMessageTimeToLive(Duration.ofSeconds(Long.parseLong(confValue)));
break;
case LOCK_DURATION:
queueDescription.setLockDuration(Duration.ofSeconds(Long.parseLong(confValue)));
break;
default:
log.error("Unknown config: [{}]", confKey);
}
});
}
public void destroy() { public void destroy() {
try { try {
client.close(); client.close();

71
common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java

@ -0,0 +1,71 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.azure.servicebus;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.HashMap;
import java.util.Map;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='service-bus'")
public class TbServiceBusQueueConfigs {
@Value("${queue.service-bus.queue-properties.core}")
private String coreProperties;
@Value("${queue.service-bus.queue-properties.rule-engine}")
private String ruleEngineProperties;
@Value("${queue.service-bus.queue-properties.transport-api}")
private String transportApiProperties;
@Value("${queue.service-bus.queue-properties.notifications}")
private String notificationsProperties;
@Value("${queue.service-bus.queue-properties.js-executor}")
private String jsExecutorProperties;
@Getter
private Map<String, String> coreConfigs;
@Getter
private Map<String, String> ruleEngineConfigs;
@Getter
private Map<String, String> transportApiConfigs;
@Getter
private Map<String, String> notificationsConfigs;
@Getter
private Map<String, String> jsExecutorConfigs;
@PostConstruct
private void init() {
coreConfigs = getConfigs(coreProperties);
ruleEngineConfigs = getConfigs(ruleEngineProperties);
transportApiConfigs = getConfigs(transportApiProperties);
notificationsConfigs = getConfigs(notificationsProperties);
jsExecutorConfigs = getConfigs(jsExecutorProperties);
}
private Map<String, String> getConfigs(String properties) {
Map<String, String> configs = new HashMap<>();
for (String property : properties.split(";")) {
int delimiterPosition = property.indexOf(":");
String key = property.substring(0, delimiterPosition);
String value = property.substring(delimiterPosition + 1);
configs.put(key, value);
}
return configs;
}
}

12
common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java

@ -30,6 +30,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
@ -61,6 +62,7 @@ public class ConsistentHashPartitionService implements PartitionService {
private final ApplicationEventPublisher applicationEventPublisher; private final ApplicationEventPublisher applicationEventPublisher;
private final TbServiceInfoProvider serviceInfoProvider; private final TbServiceInfoProvider serviceInfoProvider;
private final TenantRoutingInfoService tenantRoutingInfoService; private final TenantRoutingInfoService tenantRoutingInfoService;
private final TbQueueRuleEngineSettings tbQueueRuleEngineSettings;
private final ConcurrentMap<ServiceQueue, String> partitionTopics = new ConcurrentHashMap<>(); private final ConcurrentMap<ServiceQueue, String> partitionTopics = new ConcurrentHashMap<>();
private final ConcurrentMap<ServiceQueue, Integer> partitionSizes = new ConcurrentHashMap<>(); private final ConcurrentMap<ServiceQueue, Integer> partitionSizes = new ConcurrentHashMap<>();
private final ConcurrentMap<TenantId, TenantRoutingInfo> tenantRoutingInfoMap = new ConcurrentHashMap<>(); private final ConcurrentMap<TenantId, TenantRoutingInfo> tenantRoutingInfoMap = new ConcurrentHashMap<>();
@ -74,10 +76,14 @@ public class ConsistentHashPartitionService implements PartitionService {
private HashFunction hashFunction; private HashFunction hashFunction;
public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider, TenantRoutingInfoService tenantRoutingInfoService, ApplicationEventPublisher applicationEventPublisher) { public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider,
TenantRoutingInfoService tenantRoutingInfoService,
ApplicationEventPublisher applicationEventPublisher,
TbQueueRuleEngineSettings tbQueueRuleEngineSettings) {
this.serviceInfoProvider = serviceInfoProvider; this.serviceInfoProvider = serviceInfoProvider;
this.tenantRoutingInfoService = tenantRoutingInfoService; this.tenantRoutingInfoService = tenantRoutingInfoService;
this.applicationEventPublisher = applicationEventPublisher; this.applicationEventPublisher = applicationEventPublisher;
this.tbQueueRuleEngineSettings = tbQueueRuleEngineSettings;
} }
@PostConstruct @PostConstruct
@ -85,6 +91,10 @@ public class ConsistentHashPartitionService implements PartitionService {
this.hashFunction = forName(hashFunctionName); this.hashFunction = forName(hashFunctionName);
partitionSizes.put(new ServiceQueue(ServiceType.TB_CORE), corePartitions); partitionSizes.put(new ServiceQueue(ServiceType.TB_CORE), corePartitions);
partitionTopics.put(new ServiceQueue(ServiceType.TB_CORE), coreTopic); partitionTopics.put(new ServiceQueue(ServiceType.TB_CORE), coreTopic);
tbQueueRuleEngineSettings.getQueues().forEach(queueConfiguration -> {
partitionTopics.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queueConfiguration.getName()), queueConfiguration.getTopic());
partitionSizes.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queueConfiguration.getName()), queueConfiguration.getPartitions());
});
} }
@Override @Override

42
common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java

@ -30,11 +30,9 @@ import java.util.concurrent.TimeUnit;
public final class InMemoryStorage { public final class InMemoryStorage {
private static InMemoryStorage instance; private static InMemoryStorage instance;
private final ConcurrentHashMap<String, BlockingQueue<TbQueueMsg>> storage; private final ConcurrentHashMap<String, BlockingQueue<TbQueueMsg>> storage;
private volatile boolean stopped;
private InMemoryStorage() { private InMemoryStorage() {
storage = new ConcurrentHashMap<>(); storage = new ConcurrentHashMap<>();
stopped = false;
} }
public static InMemoryStorage getInstance() { public static InMemoryStorage getInstance() {
@ -52,33 +50,31 @@ public final class InMemoryStorage {
return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg); return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg);
} }
public <T extends TbQueueMsg> List<T> get(String topic, long durationInMillis) { public <T extends TbQueueMsg> List<T> get(String topic) throws InterruptedException {
if (storage.containsKey(topic)) { if (storage.containsKey(topic)) {
try { List<T> entities;
List<T> entities; T first = (T) storage.get(topic).poll();
T first = (T) storage.get(topic).poll(durationInMillis, TimeUnit.MILLISECONDS); if (first != null) {
if (first != null) { entities = new ArrayList<>();
entities = new ArrayList<>(); entities.add(first);
entities.add(first); List<TbQueueMsg> otherList = new ArrayList<>();
List<TbQueueMsg> otherList = new ArrayList<>(); storage.get(topic).drainTo(otherList, 999);
storage.get(topic).drainTo(otherList, 999); for (TbQueueMsg other : otherList) {
for (TbQueueMsg other : otherList) { entities.add((T) other);
entities.add((T) other);
}
} else {
entities = Collections.emptyList();
}
return entities;
} catch (InterruptedException e) {
if (!stopped) {
log.warn("Queue was interrupted", e);
} }
} else {
entities = Collections.emptyList();
} }
return entities;
} }
return Collections.emptyList(); return Collections.emptyList();
} }
public void stop() { /**
stopped = true; * Used primarily for testing.
*/
public void cleanup() {
storage.clear();
} }
} }

11
common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java

@ -66,7 +66,16 @@ public class InMemoryTbQueueConsumer<T extends TbQueueMsg> implements TbQueueCon
if (subscribed) { if (subscribed) {
List<T> messages = partitions List<T> messages = partitions
.stream() .stream()
.map(tpi -> storage.get(tpi.getFullTopicName(), durationInMillis)) .map(tpi -> {
try {
return storage.get(tpi.getFullTopicName());
} catch (InterruptedException e) {
if (!stopped) {
log.error("Queue was interrupted.", e);
}
return Collections.emptyList();
}
})
.flatMap(List::stream) .flatMap(List::stream)
.map(msg -> (T) msg).collect(Collectors.toList()); .map(msg -> (T) msg).collect(Collectors.toList());
if (messages.size() > 0) { if (messages.size() > 0) {

11
common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java

@ -30,6 +30,7 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin;
@ -47,8 +48,10 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory {
private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbAwsSqsSettings sqsSettings; private final TbAwsSqsSettings sqsSettings;
private final TbQueueCoreSettings coreSettings;
private final TbServiceInfoProvider serviceInfoProvider; private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin notificationAdmin;
@ -56,12 +59,15 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory {
TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueTransportNotificationSettings transportNotificationSettings,
TbAwsSqsSettings sqsSettings, TbAwsSqsSettings sqsSettings,
TbServiceInfoProvider serviceInfoProvider, TbServiceInfoProvider serviceInfoProvider,
TbQueueCoreSettings coreSettings,
TbAwsSqsQueueAttributes sqsQueueAttributes) { TbAwsSqsQueueAttributes sqsQueueAttributes) {
this.transportApiSettings = transportApiSettings; this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings; this.transportNotificationSettings = transportNotificationSettings;
this.sqsSettings = sqsSettings; this.sqsSettings = sqsSettings;
this.serviceInfoProvider = serviceInfoProvider; this.serviceInfoProvider = serviceInfoProvider;
this.coreSettings = coreSettings;
this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes());
this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes());
this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes());
} }
@ -94,7 +100,7 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory {
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic()); return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic());
} }
@Override @Override
@ -105,6 +111,9 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory {
@PreDestroy @PreDestroy
private void destroy() { private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (transportApiAdmin != null) { if (transportApiAdmin != null) {
transportApiAdmin.destroy(); transportApiAdmin.destroy();
} }

9
common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java

@ -32,9 +32,9 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer;
import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin;
@Component @Component
@ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')")
@ -43,13 +43,16 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory
private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbServiceInfoProvider serviceInfoProvider; private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueCoreSettings coreSettings;
public InMemoryTbTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, public InMemoryTbTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings,
TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueTransportNotificationSettings transportNotificationSettings,
TbServiceInfoProvider serviceInfoProvider) { TbServiceInfoProvider serviceInfoProvider,
TbQueueCoreSettings coreSettings) {
this.transportApiSettings = transportApiSettings; this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings; this.transportNotificationSettings = transportNotificationSettings;
this.serviceInfoProvider = serviceInfoProvider; this.serviceInfoProvider = serviceInfoProvider;
this.coreSettings = coreSettings;
} }
@Override @Override
@ -86,7 +89,7 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); return new InMemoryTbQueueProducer<>(coreSettings.getTopic());
} }
@Override @Override

20
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java

@ -49,6 +49,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import javax.annotation.PreDestroy;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
@Component @Component
@ -247,4 +248,23 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
return builder.build(); return builder.build();
} }
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
} }

19
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java

@ -47,6 +47,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import javax.annotation.PreDestroy;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
@Component @Component
@ -218,4 +219,22 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
return builder.build(); return builder.build();
} }
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
} }

17
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java

@ -45,6 +45,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import javax.annotation.PreDestroy;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
@Component @Component
@ -190,4 +191,20 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
return builder.build(); return builder.build();
} }
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
} }

18
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java

@ -40,6 +40,8 @@ import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import javax.annotation.PreDestroy;
@Component @Component
@ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')")
@Slf4j @Slf4j
@ -135,4 +137,20 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory {
responseBuilder.admin(notificationAdmin); responseBuilder.admin(notificationAdmin);
return responseBuilder.build(); return responseBuilder.build();
} }
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
} }

64
common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java

@ -42,6 +42,7 @@ import org.thingsboard.server.queue.pubsub.TbPubSubAdmin;
import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate;
import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate;
import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSettings;
import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
@ -49,6 +50,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import javax.annotation.PreDestroy;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
@Component @Component
@ -60,11 +62,16 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueRuleEngineSettings ruleEngineSettings;
private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbQueueAdmin admin;
private final PartitionService partitionService; private final PartitionService partitionService;
private final TbServiceInfoProvider serviceInfoProvider; private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin jsExecutorAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings, public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings,
TbQueueCoreSettings coreSettings, TbQueueCoreSettings coreSettings,
TbQueueRuleEngineSettings ruleEngineSettings, TbQueueRuleEngineSettings ruleEngineSettings,
@ -72,86 +79,92 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueTransportNotificationSettings transportNotificationSettings,
PartitionService partitionService, PartitionService partitionService,
TbServiceInfoProvider serviceInfoProvider, TbServiceInfoProvider serviceInfoProvider,
TbPubSubSubscriptionSettings pubSubSubscriptionSettings,
TbQueueRemoteJsInvokeSettings jsInvokeSettings) { TbQueueRemoteJsInvokeSettings jsInvokeSettings) {
this.pubSubSettings = pubSubSettings; this.pubSubSettings = pubSubSettings;
this.coreSettings = coreSettings; this.coreSettings = coreSettings;
this.ruleEngineSettings = ruleEngineSettings; this.ruleEngineSettings = ruleEngineSettings;
this.transportApiSettings = transportApiSettings; this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings; this.transportNotificationSettings = transportNotificationSettings;
this.admin = new TbPubSubAdmin(pubSubSettings);
this.partitionService = partitionService; this.partitionService = partitionService;
this.serviceInfoProvider = serviceInfoProvider; this.serviceInfoProvider = serviceInfoProvider;
this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings());
this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings());
this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings());
this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings());
this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings());
this.jsInvokeSettings = jsInvokeSettings; this.jsInvokeSettings = jsInvokeSettings;
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic(), return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings,
partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, coreSettings.getTopic(), return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings,
partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() { public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic(), return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getResponsesTopic()); return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic());
} }
@Override @Override
@Bean @Bean
public TbQueueRequestTemplate<TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> createRemoteJsRequestTemplate() { public TbQueueRequestTemplate<TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> createRemoteJsRequestTemplate() {
TbQueueProducer<TbProtoJsQueueMsg<RemoteJsRequest>> producer = new TbPubSubProducerTemplate<>(admin, pubSubSettings, jsInvokeSettings.getRequestTopic()); TbQueueProducer<TbProtoJsQueueMsg<RemoteJsRequest>> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic());
TbQueueConsumer<TbProtoQueueMsg<RemoteJsResponse>> consumer = new TbPubSubConsumerTemplate<>(admin, pubSubSettings, TbQueueConsumer<TbProtoQueueMsg<RemoteJsResponse>> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings,
jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(),
msg -> { msg -> {
RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder(); RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder();
@ -161,7 +174,7 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
<TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder(); <TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder();
builder.queueAdmin(admin); builder.queueAdmin(jsExecutorAdmin);
builder.requestTemplate(producer); builder.requestTemplate(producer);
builder.responseTemplate(consumer); builder.responseTemplate(consumer);
builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests());
@ -169,4 +182,23 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
return builder.build(); return builder.build();
} }
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
} }

54
common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java

@ -30,6 +30,8 @@ import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.pubsub.TbPubSubAdmin;
import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
@ -40,6 +42,8 @@ import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate;
import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate;
import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSettings;
import javax.annotation.PreDestroy;
@Component @Component
@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'") @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'")
public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory {
@ -47,75 +51,99 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory {
private final TbPubSubSettings pubSubSettings; private final TbPubSubSettings pubSubSettings;
private final TbQueueCoreSettings coreSettings; private final TbQueueCoreSettings coreSettings;
private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueAdmin admin;
private final PartitionService partitionService; private final PartitionService partitionService;
private final TbServiceInfoProvider serviceInfoProvider; private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin jsExecutorAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
public PubSubTbCoreQueueFactory(TbPubSubSettings pubSubSettings, public PubSubTbCoreQueueFactory(TbPubSubSettings pubSubSettings,
TbQueueCoreSettings coreSettings, TbQueueCoreSettings coreSettings,
TbQueueTransportApiSettings transportApiSettings, TbQueueTransportApiSettings transportApiSettings,
TbQueueAdmin admin,
PartitionService partitionService, PartitionService partitionService,
TbServiceInfoProvider serviceInfoProvider) { TbServiceInfoProvider serviceInfoProvider,
TbPubSubSubscriptionSettings pubSubSubscriptionSettings) {
this.pubSubSettings = pubSubSettings; this.pubSubSettings = pubSubSettings;
this.coreSettings = coreSettings; this.coreSettings = coreSettings;
this.transportApiSettings = transportApiSettings; this.transportApiSettings = transportApiSettings;
this.admin = admin;
this.partitionService = partitionService; this.partitionService = partitionService;
this.serviceInfoProvider = serviceInfoProvider; this.serviceInfoProvider = serviceInfoProvider;
this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings());
this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings());
this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings());
this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, coreSettings.getTopic(), return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings,
partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() { public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic(), return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() { public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
return null; return null;
} }
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
} }

50
common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java

@ -32,13 +32,17 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.pubsub.TbPubSubAdmin;
import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate;
import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate;
import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSettings;
import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import javax.annotation.PreDestroy;
@Component @Component
@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'") @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'")
public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
@ -46,59 +50,67 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory
private final TbPubSubSettings pubSubSettings; private final TbPubSubSettings pubSubSettings;
private final TbQueueCoreSettings coreSettings; private final TbQueueCoreSettings coreSettings;
private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueRuleEngineSettings ruleEngineSettings;
private final TbQueueAdmin admin;
private final PartitionService partitionService; private final PartitionService partitionService;
private final TbServiceInfoProvider serviceInfoProvider; private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin jsExecutorAdmin;
private final TbQueueAdmin notificationAdmin;
public PubSubTbRuleEngineQueueFactory(TbPubSubSettings pubSubSettings, public PubSubTbRuleEngineQueueFactory(TbPubSubSettings pubSubSettings,
TbQueueCoreSettings coreSettings, TbQueueCoreSettings coreSettings,
TbQueueRuleEngineSettings ruleEngineSettings, TbQueueRuleEngineSettings ruleEngineSettings,
TbQueueAdmin admin,
PartitionService partitionService, PartitionService partitionService,
TbServiceInfoProvider serviceInfoProvider) { TbServiceInfoProvider serviceInfoProvider,
TbPubSubSubscriptionSettings pubSubSubscriptionSettings) {
this.pubSubSettings = pubSubSettings; this.pubSubSettings = pubSubSettings;
this.coreSettings = coreSettings; this.coreSettings = coreSettings;
this.ruleEngineSettings = ruleEngineSettings; this.ruleEngineSettings = ruleEngineSettings;
this.admin = admin;
this.partitionService = partitionService; this.partitionService = partitionService;
this.serviceInfoProvider = serviceInfoProvider; this.serviceInfoProvider = serviceInfoProvider;
this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings());
this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings());
this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings());
this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic(), return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings,
partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@ -107,4 +119,20 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory
public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() { public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
return null; return null;
} }
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
} }

54
common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java

@ -25,12 +25,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
@ -38,6 +34,13 @@ import org.thingsboard.server.queue.pubsub.TbPubSubAdmin;
import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate;
import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate;
import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSettings;
import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import javax.annotation.PreDestroy;
@Component @Component
@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')")
@ -50,33 +53,42 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory {
private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueRuleEngineSettings ruleEngineSettings;
private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbQueueAdmin admin;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
public PubSubTransportQueueFactory(TbPubSubSettings pubSubSettings, public PubSubTransportQueueFactory(TbPubSubSettings pubSubSettings,
TbServiceInfoProvider serviceInfoProvider, TbServiceInfoProvider serviceInfoProvider,
TbQueueCoreSettings coreSettings, TbQueueCoreSettings coreSettings,
TbQueueRuleEngineSettings ruleEngineSettings, TbQueueRuleEngineSettings ruleEngineSettings,
TbQueueTransportApiSettings transportApiSettings, TbQueueTransportApiSettings transportApiSettings,
TbQueueTransportNotificationSettings transportNotificationSettings) { TbQueueTransportNotificationSettings transportNotificationSettings,
TbPubSubSubscriptionSettings pubSubSubscriptionSettings) {
this.pubSubSettings = pubSubSettings; this.pubSubSettings = pubSubSettings;
this.serviceInfoProvider = serviceInfoProvider; this.serviceInfoProvider = serviceInfoProvider;
this.coreSettings = coreSettings; this.coreSettings = coreSettings;
this.ruleEngineSettings = ruleEngineSettings; this.ruleEngineSettings = ruleEngineSettings;
this.transportApiSettings = transportApiSettings; this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings; this.transportNotificationSettings = transportNotificationSettings;
this.admin = new TbPubSubAdmin(pubSubSettings);
this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings());
this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings());
this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings());
this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings());
} }
@Override @Override
public TbQueueRequestTemplate<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiRequestTemplate() { public TbQueueRequestTemplate<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiRequestTemplate() {
TbQueueProducer<TbProtoQueueMsg<TransportApiRequestMsg>> producer = new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic()); TbQueueProducer<TbProtoQueueMsg<TransportApiRequestMsg>> producer = new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic());
TbQueueConsumer<TbProtoQueueMsg<TransportApiResponseMsg>> consumer = new TbPubSubConsumerTemplate<>(admin, pubSubSettings, TbQueueConsumer<TbProtoQueueMsg<TransportApiResponseMsg>> consumer = new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings,
transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders()));
DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> templateBuilder = DefaultTbQueueRequestTemplate.builder(); <TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> templateBuilder = DefaultTbQueueRequestTemplate.builder();
templateBuilder.queueAdmin(admin); templateBuilder.queueAdmin(transportApiAdmin);
templateBuilder.requestTemplate(producer); templateBuilder.requestTemplate(producer);
templateBuilder.responseTemplate(consumer); templateBuilder.responseTemplate(consumer);
templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests());
@ -87,18 +99,34 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory {
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsConsumer() {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings,
transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
} }

71
common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java

@ -32,10 +32,10 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin;
import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
@ -44,6 +44,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import javax.annotation.PreDestroy;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
@Component @Component
@ -57,9 +58,13 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbRabbitMqSettings rabbitMqSettings; private final TbRabbitMqSettings rabbitMqSettings;
private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
private final TbQueueAdmin admin; private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin jsExecutorAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
public RabbitMqMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, public RabbitMqMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings,
TbQueueRuleEngineSettings ruleEngineSettings, TbQueueRuleEngineSettings ruleEngineSettings,
@ -67,8 +72,8 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
TbQueueTransportApiSettings transportApiSettings, TbQueueTransportApiSettings transportApiSettings,
TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueTransportNotificationSettings transportNotificationSettings,
TbRabbitMqSettings rabbitMqSettings, TbRabbitMqSettings rabbitMqSettings,
TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbRabbitMqQueueArguments queueArguments,
TbQueueAdmin admin) { TbQueueRemoteJsInvokeSettings jsInvokeSettings) {
this.partitionService = partitionService; this.partitionService = partitionService;
this.coreSettings = coreSettings; this.coreSettings = coreSettings;
this.serviceInfoProvider = serviceInfoProvider; this.serviceInfoProvider = serviceInfoProvider;
@ -76,77 +81,82 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
this.transportApiSettings = transportApiSettings; this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings; this.transportNotificationSettings = transportNotificationSettings;
this.rabbitMqSettings = rabbitMqSettings; this.rabbitMqSettings = rabbitMqSettings;
this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs());
this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs());
this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs());
this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs());
this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs());
this.jsInvokeSettings = jsInvokeSettings; this.jsInvokeSettings = jsInvokeSettings;
this.admin = admin;
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToTransportMsg>> createTransportNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic()); return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> createRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> createTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> createTbCoreMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic(), return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings,
partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> createToCoreMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> createToCoreMsgConsumer() {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic(), return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings,
partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>> createTransportApiRequestConsumer() { public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>> createTransportApiRequestConsumer() {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> createTransportApiResponseProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> createTransportApiResponseProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getResponsesTopic());
} }
@Override @Override
@Bean @Bean
public TbQueueRequestTemplate<TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> createRemoteJsRequestTemplate() { public TbQueueRequestTemplate<TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> createRemoteJsRequestTemplate() {
TbQueueProducer<TbProtoJsQueueMsg<RemoteJsRequest>> producer = new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, jsInvokeSettings.getRequestTopic()); TbQueueProducer<TbProtoJsQueueMsg<RemoteJsRequest>> producer = new TbRabbitMqProducerTemplate<>(jsExecutorAdmin, rabbitMqSettings, jsInvokeSettings.getRequestTopic());
TbQueueConsumer<TbProtoQueueMsg<RemoteJsResponse>> consumer = new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, TbQueueConsumer<TbProtoQueueMsg<RemoteJsResponse>> consumer = new TbRabbitMqConsumerTemplate<>(jsExecutorAdmin, rabbitMqSettings,
jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(),
msg -> { msg -> {
RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder(); RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder();
@ -156,7 +166,7 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
<TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder(); <TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder();
builder.queueAdmin(admin); builder.queueAdmin(jsExecutorAdmin);
builder.requestTemplate(producer); builder.requestTemplate(producer);
builder.responseTemplate(consumer); builder.responseTemplate(consumer);
builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests());
@ -164,4 +174,23 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
return builder.build(); return builder.build();
} }
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
} }

57
common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java

@ -34,13 +34,17 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import javax.annotation.PreDestroy;
@Component @Component
@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-core'") @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-core'")
public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory {
@ -51,7 +55,12 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory {
private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportApiSettings transportApiSettings;
private final PartitionService partitionService; private final PartitionService partitionService;
private final TbServiceInfoProvider serviceInfoProvider; private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueAdmin admin;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin jsExecutorAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
public RabbitMqTbCoreQueueFactory(TbRabbitMqSettings rabbitMqSettings, public RabbitMqTbCoreQueueFactory(TbRabbitMqSettings rabbitMqSettings,
TbQueueCoreSettings coreSettings, TbQueueCoreSettings coreSettings,
@ -59,67 +68,91 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory {
TbQueueRuleEngineSettings ruleEngineSettings, TbQueueRuleEngineSettings ruleEngineSettings,
PartitionService partitionService, PartitionService partitionService,
TbServiceInfoProvider serviceInfoProvider, TbServiceInfoProvider serviceInfoProvider,
TbQueueAdmin admin) { TbRabbitMqQueueArguments queueArguments) {
this.rabbitMqSettings = rabbitMqSettings; this.rabbitMqSettings = rabbitMqSettings;
this.coreSettings = coreSettings; this.coreSettings = coreSettings;
this.transportApiSettings = transportApiSettings; this.transportApiSettings = transportApiSettings;
this.ruleEngineSettings = ruleEngineSettings; this.ruleEngineSettings = ruleEngineSettings;
this.partitionService = partitionService; this.partitionService = partitionService;
this.serviceInfoProvider = serviceInfoProvider; this.serviceInfoProvider = serviceInfoProvider;
this.admin = admin;
this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs());
this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs());
this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs());
this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs());
this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic(), return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings,
partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() { public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() { public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
return null; return null;
} }
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
} }

48
common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java

@ -32,13 +32,17 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import javax.annotation.PreDestroy;
@Component @Component
@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-rule-engine'") @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-rule-engine'")
public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
@ -48,55 +52,63 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor
private final TbServiceInfoProvider serviceInfoProvider; private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueRuleEngineSettings ruleEngineSettings;
private final TbRabbitMqSettings rabbitMqSettings; private final TbRabbitMqSettings rabbitMqSettings;
private final TbQueueAdmin admin;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin jsExecutorAdmin;
private final TbQueueAdmin notificationAdmin;
public RabbitMqTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, public RabbitMqTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings,
TbQueueRuleEngineSettings ruleEngineSettings, TbQueueRuleEngineSettings ruleEngineSettings,
TbServiceInfoProvider serviceInfoProvider, TbServiceInfoProvider serviceInfoProvider,
TbRabbitMqSettings rabbitMqSettings, TbRabbitMqSettings rabbitMqSettings,
TbQueueAdmin admin) { TbRabbitMqQueueArguments queueArguments) {
this.partitionService = partitionService; this.partitionService = partitionService;
this.coreSettings = coreSettings; this.coreSettings = coreSettings;
this.serviceInfoProvider = serviceInfoProvider; this.serviceInfoProvider = serviceInfoProvider;
this.ruleEngineSettings = ruleEngineSettings; this.ruleEngineSettings = ruleEngineSettings;
this.rabbitMqSettings = rabbitMqSettings; this.rabbitMqSettings = rabbitMqSettings;
this.admin = admin;
this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs());
this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs());
this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs());
this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic(), return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings,
partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@ -105,4 +117,20 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor
public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() { public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
return null; return null;
} }
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
} }

50
common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java

@ -30,12 +30,17 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import javax.annotation.PreDestroy;
@Component @Component
@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')")
@Slf4j @Slf4j
@ -43,34 +48,45 @@ public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory {
private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbRabbitMqSettings rabbitMqSettings; private final TbRabbitMqSettings rabbitMqSettings;
private final TbQueueAdmin admin;
private final TbServiceInfoProvider serviceInfoProvider; private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueCoreSettings coreSettings;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
public RabbitMqTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, public RabbitMqTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings,
TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueTransportNotificationSettings transportNotificationSettings,
TbRabbitMqSettings rabbitMqSettings, TbRabbitMqSettings rabbitMqSettings,
TbServiceInfoProvider serviceInfoProvider, TbServiceInfoProvider serviceInfoProvider,
TbQueueAdmin admin) { TbQueueCoreSettings coreSettings,
TbRabbitMqQueueArguments queueArguments) {
this.transportApiSettings = transportApiSettings; this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings; this.transportNotificationSettings = transportNotificationSettings;
this.rabbitMqSettings = rabbitMqSettings; this.rabbitMqSettings = rabbitMqSettings;
this.admin = admin;
this.serviceInfoProvider = serviceInfoProvider; this.serviceInfoProvider = serviceInfoProvider;
this.coreSettings = coreSettings;
this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs());
this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs());
this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs());
this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs());
} }
@Override @Override
public TbQueueRequestTemplate<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiRequestTemplate() { public TbQueueRequestTemplate<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiRequestTemplate() {
TbRabbitMqProducerTemplate<TbProtoQueueMsg<TransportApiRequestMsg>> producerTemplate = TbRabbitMqProducerTemplate<TbProtoQueueMsg<TransportApiRequestMsg>> producerTemplate =
new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic());
TbRabbitMqConsumerTemplate<TbProtoQueueMsg<TransportApiResponseMsg>> consumerTemplate = TbRabbitMqConsumerTemplate<TbProtoQueueMsg<TransportApiResponseMsg>> consumerTemplate =
new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings,
transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders()));
DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> templateBuilder = DefaultTbQueueRequestTemplate.builder(); <TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> templateBuilder = DefaultTbQueueRequestTemplate.builder();
templateBuilder.queueAdmin(admin); templateBuilder.queueAdmin(transportApiAdmin);
templateBuilder.requestTemplate(producerTemplate); templateBuilder.requestTemplate(producerTemplate);
templateBuilder.responseTemplate(consumerTemplate); templateBuilder.responseTemplate(consumerTemplate);
templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests());
@ -81,17 +97,33 @@ public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory {
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsConsumer() {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
} }

63
common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java

@ -30,19 +30,25 @@ import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings;
import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.kafka.TbKafkaAdmin;
import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import javax.annotation.PreDestroy;
@Component @Component
@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'") @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'")
public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory {
@ -54,7 +60,12 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul
private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbServiceBusSettings serviceBusSettings; private final TbServiceBusSettings serviceBusSettings;
private final TbQueueAdmin admin;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin jsExecutorAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
public ServiceBusMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, public ServiceBusMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings,
TbQueueRuleEngineSettings ruleEngineSettings, TbQueueRuleEngineSettings ruleEngineSettings,
@ -62,7 +73,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul
TbQueueTransportApiSettings transportApiSettings, TbQueueTransportApiSettings transportApiSettings,
TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueTransportNotificationSettings transportNotificationSettings,
TbServiceBusSettings serviceBusSettings, TbServiceBusSettings serviceBusSettings,
TbQueueAdmin admin) { TbServiceBusQueueConfigs serviceBusQueueConfigs) {
this.partitionService = partitionService; this.partitionService = partitionService;
this.coreSettings = coreSettings; this.coreSettings = coreSettings;
this.serviceInfoProvider = serviceInfoProvider; this.serviceInfoProvider = serviceInfoProvider;
@ -70,73 +81,97 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul
this.transportApiSettings = transportApiSettings; this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings; this.transportNotificationSettings = transportNotificationSettings;
this.serviceBusSettings = serviceBusSettings; this.serviceBusSettings = serviceBusSettings;
this.admin = admin;
this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs());
this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs());
this.jsExecutorAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getJsExecutorConfigs());
this.transportApiAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getTransportApiConfigs());
this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic()); return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(), return new TbServiceBusConsumerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings,
partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(), return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings,
partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() { public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(), return new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getResponsesTopic()); return new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getResponsesTopic());
} }
@Override @Override
public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() { public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
return null; return null;
} }
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
} }

57
common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java

@ -29,8 +29,10 @@ import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings;
import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -40,6 +42,8 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import javax.annotation.PreDestroy;
@Component @Component
@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'") @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'")
public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory {
@ -50,7 +54,12 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory {
private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportApiSettings transportApiSettings;
private final PartitionService partitionService; private final PartitionService partitionService;
private final TbServiceInfoProvider serviceInfoProvider; private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueAdmin admin;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin jsExecutorAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
public ServiceBusTbCoreQueueFactory(TbServiceBusSettings serviceBusSettings, public ServiceBusTbCoreQueueFactory(TbServiceBusSettings serviceBusSettings,
TbQueueCoreSettings coreSettings, TbQueueCoreSettings coreSettings,
@ -58,67 +67,91 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory {
TbQueueRuleEngineSettings ruleEngineSettings, TbQueueRuleEngineSettings ruleEngineSettings,
PartitionService partitionService, PartitionService partitionService,
TbServiceInfoProvider serviceInfoProvider, TbServiceInfoProvider serviceInfoProvider,
TbQueueAdmin admin) { TbServiceBusQueueConfigs serviceBusQueueConfigs) {
this.serviceBusSettings = serviceBusSettings; this.serviceBusSettings = serviceBusSettings;
this.coreSettings = coreSettings; this.coreSettings = coreSettings;
this.transportApiSettings = transportApiSettings; this.transportApiSettings = transportApiSettings;
this.ruleEngineSettings = ruleEngineSettings; this.ruleEngineSettings = ruleEngineSettings;
this.partitionService = partitionService; this.partitionService = partitionService;
this.serviceInfoProvider = serviceInfoProvider; this.serviceInfoProvider = serviceInfoProvider;
this.admin = admin;
this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs());
this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs());
this.jsExecutorAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getJsExecutorConfigs());
this.transportApiAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getTransportApiConfigs());
this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(), return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings,
partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() { public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(), return new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() { public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
return null; return null;
} }
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
} }

48
common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java

@ -27,8 +27,10 @@ import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings;
import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -38,6 +40,8 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import javax.annotation.PreDestroy;
@Component @Component
@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-rule-engine'") @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-rule-engine'")
public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
@ -47,55 +51,63 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact
private final TbServiceInfoProvider serviceInfoProvider; private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueRuleEngineSettings ruleEngineSettings;
private final TbServiceBusSettings serviceBusSettings; private final TbServiceBusSettings serviceBusSettings;
private final TbQueueAdmin admin;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin jsExecutorAdmin;
private final TbQueueAdmin notificationAdmin;
public ServiceBusTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, public ServiceBusTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings,
TbQueueRuleEngineSettings ruleEngineSettings, TbQueueRuleEngineSettings ruleEngineSettings,
TbServiceInfoProvider serviceInfoProvider, TbServiceInfoProvider serviceInfoProvider,
TbServiceBusSettings serviceBusSettings, TbServiceBusSettings serviceBusSettings,
TbQueueAdmin admin) { TbServiceBusQueueConfigs serviceBusQueueConfigs) {
this.partitionService = partitionService; this.partitionService = partitionService;
this.coreSettings = coreSettings; this.coreSettings = coreSettings;
this.serviceInfoProvider = serviceInfoProvider; this.serviceInfoProvider = serviceInfoProvider;
this.ruleEngineSettings = ruleEngineSettings; this.ruleEngineSettings = ruleEngineSettings;
this.serviceBusSettings = serviceBusSettings; this.serviceBusSettings = serviceBusSettings;
this.admin = admin;
this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs());
this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs());
this.jsExecutorAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getJsExecutorConfigs());
this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(), return new TbServiceBusConsumerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() { public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings,
partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@ -104,4 +116,20 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact
public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() { public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
return null; return null;
} }
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
} }

51
common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java

@ -23,15 +23,20 @@ import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs;
import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import javax.annotation.PreDestroy;
@Component @Component
@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')")
@Slf4j @Slf4j
@ -39,34 +44,43 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory
private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbServiceBusSettings serviceBusSettings; private final TbServiceBusSettings serviceBusSettings;
private final TbQueueAdmin admin;
private final TbServiceInfoProvider serviceInfoProvider; private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueCoreSettings coreSettings;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
public ServiceBusTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, public ServiceBusTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings,
TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueTransportNotificationSettings transportNotificationSettings,
TbServiceBusSettings serviceBusSettings, TbServiceBusSettings serviceBusSettings,
TbServiceInfoProvider serviceInfoProvider, TbServiceInfoProvider serviceInfoProvider,
TbQueueAdmin admin) { TbQueueCoreSettings coreSettings,
TbServiceBusQueueConfigs serviceBusQueueConfigs) {
this.transportApiSettings = transportApiSettings; this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings; this.transportNotificationSettings = transportNotificationSettings;
this.serviceBusSettings = serviceBusSettings; this.serviceBusSettings = serviceBusSettings;
this.admin = admin;
this.serviceInfoProvider = serviceInfoProvider; this.serviceInfoProvider = serviceInfoProvider;
this.coreSettings = coreSettings;
this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs());
this.transportApiAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getTransportApiConfigs());
this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs());
} }
@Override @Override
public TbQueueRequestTemplate<TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>, TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> createTransportApiRequestTemplate() { public TbQueueRequestTemplate<TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>, TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> createTransportApiRequestTemplate() {
TbQueueProducer<TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>> producerTemplate = TbQueueProducer<TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>> producerTemplate =
new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic());
TbQueueConsumer<TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> consumerTemplate = TbQueueConsumer<TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> consumerTemplate =
new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings,
transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders()));
DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
<TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>, TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> templateBuilder = DefaultTbQueueRequestTemplate.builder(); <TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>, TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> templateBuilder = DefaultTbQueueRequestTemplate.builder();
templateBuilder.queueAdmin(admin); templateBuilder.queueAdmin(transportApiAdmin);
templateBuilder.requestTemplate(producerTemplate); templateBuilder.requestTemplate(producerTemplate);
templateBuilder.responseTemplate(consumerTemplate); templateBuilder.responseTemplate(consumerTemplate);
templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests());
@ -77,18 +91,31 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> createRuleEngineMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); return new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic());
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> createTbCoreMsgProducer() { public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> createTbCoreMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToTransportMsg>> createTransportNotificationsConsumer() { public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToTransportMsg>> createTransportNotificationsConsumer() {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings,
transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders()));
} }
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
} }

120
common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java

@ -19,33 +19,40 @@ import com.google.cloud.pubsub.v1.SubscriptionAdminClient;
import com.google.cloud.pubsub.v1.SubscriptionAdminSettings; import com.google.cloud.pubsub.v1.SubscriptionAdminSettings;
import com.google.cloud.pubsub.v1.TopicAdminClient; import com.google.cloud.pubsub.v1.TopicAdminClient;
import com.google.cloud.pubsub.v1.TopicAdminSettings; import com.google.cloud.pubsub.v1.TopicAdminSettings;
import com.google.protobuf.Duration;
import com.google.pubsub.v1.ListSubscriptionsRequest; import com.google.pubsub.v1.ListSubscriptionsRequest;
import com.google.pubsub.v1.ListTopicsRequest; import com.google.pubsub.v1.ListTopicsRequest;
import com.google.pubsub.v1.ProjectName; import com.google.pubsub.v1.ProjectName;
import com.google.pubsub.v1.ProjectSubscriptionName; import com.google.pubsub.v1.ProjectSubscriptionName;
import com.google.pubsub.v1.ProjectTopicName; import com.google.pubsub.v1.ProjectTopicName;
import com.google.pubsub.v1.PushConfig;
import com.google.pubsub.v1.Subscription; import com.google.pubsub.v1.Subscription;
import com.google.pubsub.v1.Topic; import com.google.pubsub.v1.Topic;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
import java.io.IOException; import java.io.IOException;
import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
@Slf4j @Slf4j
public class TbPubSubAdmin implements TbQueueAdmin { public class TbPubSubAdmin implements TbQueueAdmin {
private static final String ACK_DEADLINE = "ackDeadlineInSec";
private static final String MESSAGE_RETENTION = "messageRetentionInSec";
private final TopicAdminClient topicAdminClient;
private final SubscriptionAdminClient subscriptionAdminClient;
private final TbPubSubSettings pubSubSettings; private final TbPubSubSettings pubSubSettings;
private final SubscriptionAdminSettings subscriptionAdminSettings;
private final TopicAdminSettings topicAdminSettings;
private final Set<String> topicSet = ConcurrentHashMap.newKeySet(); private final Set<String> topicSet = ConcurrentHashMap.newKeySet();
private final Set<String> subscriptionSet = ConcurrentHashMap.newKeySet(); private final Set<String> subscriptionSet = ConcurrentHashMap.newKeySet();
private final Map<String, String> subscriptionProperties;
public TbPubSubAdmin(TbPubSubSettings pubSubSettings) { public TbPubSubAdmin(TbPubSubSettings pubSubSettings, Map<String, String> subscriptionSettings) {
this.pubSubSettings = pubSubSettings; this.pubSubSettings = pubSubSettings;
this.subscriptionProperties = subscriptionSettings;
TopicAdminSettings topicAdminSettings;
try { try {
topicAdminSettings = TopicAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); topicAdminSettings = TopicAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build();
} catch (IOException e) { } catch (IOException e) {
@ -53,6 +60,7 @@ public class TbPubSubAdmin implements TbQueueAdmin {
throw new RuntimeException("Failed to create TopicAdminSettings."); throw new RuntimeException("Failed to create TopicAdminSettings.");
} }
SubscriptionAdminSettings subscriptionAdminSettings;
try { try {
subscriptionAdminSettings = SubscriptionAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); subscriptionAdminSettings = SubscriptionAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build();
} catch (IOException e) { } catch (IOException e) {
@ -60,7 +68,9 @@ public class TbPubSubAdmin implements TbQueueAdmin {
throw new RuntimeException("Failed to create SubscriptionAdminSettings."); throw new RuntimeException("Failed to create SubscriptionAdminSettings.");
} }
try (TopicAdminClient topicAdminClient = TopicAdminClient.create(topicAdminSettings)) { try {
topicAdminClient = TopicAdminClient.create(topicAdminSettings);
ListTopicsRequest listTopicsRequest = ListTopicsRequest listTopicsRequest =
ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build();
TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest);
@ -72,7 +82,8 @@ public class TbPubSubAdmin implements TbQueueAdmin {
throw new RuntimeException("Failed to get topics.", e); throw new RuntimeException("Failed to get topics.", e);
} }
try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings)) { try {
subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings);
ListSubscriptionsRequest listSubscriptionsRequest = ListSubscriptionsRequest listSubscriptionsRequest =
ListSubscriptionsRequest.newBuilder() ListSubscriptionsRequest.newBuilder()
@ -99,31 +110,21 @@ public class TbPubSubAdmin implements TbQueueAdmin {
return; return;
} }
try (TopicAdminClient topicAdminClient = TopicAdminClient.create(topicAdminSettings)) { ListTopicsRequest listTopicsRequest =
ListTopicsRequest listTopicsRequest = ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build();
ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest);
TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); for (Topic topic : response.iterateAll()) {
for (Topic topic : response.iterateAll()) { if (topic.getName().contains(topicName.toString())) {
if (topic.getName().contains(topicName.toString())) { topicSet.add(topic.getName());
topicSet.add(topic.getName()); createSubscriptionIfNotExists(partition, topicName);
createSubscriptionIfNotExists(partition, topicName); return;
return;
}
} }
topicAdminClient.createTopic(topicName);
topicSet.add(topicName.toString());
log.info("Created new topic: [{}]", topicName.toString());
createSubscriptionIfNotExists(partition, topicName);
} catch (IOException e) {
log.error("Failed to create topic: [{}].", topicName.toString(), e);
throw new RuntimeException("Failed to create topic.", e);
} }
}
@Override
public void destroy() {
topicAdminClient.createTopic(topicName);
topicSet.add(topicName.toString());
log.info("Created new topic: [{}]", topicName.toString());
createSubscriptionIfNotExists(partition, topicName);
} }
private void createSubscriptionIfNotExists(String partition, ProjectTopicName topicName) { private void createSubscriptionIfNotExists(String partition, ProjectTopicName topicName) {
@ -134,29 +135,52 @@ public class TbPubSubAdmin implements TbQueueAdmin {
return; return;
} }
try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings)) { ListSubscriptionsRequest listSubscriptionsRequest =
ListSubscriptionsRequest listSubscriptionsRequest = ListSubscriptionsRequest.newBuilder().setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()).build();
ListSubscriptionsRequest.newBuilder() SubscriptionAdminClient.ListSubscriptionsPagedResponse response = subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest);
.setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()) for (Subscription subscription : response.iterateAll()) {
.build(); if (subscription.getName().equals(subscriptionName.toString())) {
SubscriptionAdminClient.ListSubscriptionsPagedResponse response = subscriptionSet.add(subscription.getName());
subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); return;
for (Subscription subscription : response.iterateAll()) {
if (subscription.getName().equals(subscriptionName.toString())) {
subscriptionSet.add(subscription.getName());
return;
}
} }
}
subscriptionAdminClient.createSubscription( Subscription.Builder subscriptionBuilder = Subscription
subscriptionName, topicName, PushConfig.getDefaultInstance(), pubSubSettings.getAckDeadline()).getName(); .newBuilder()
subscriptionSet.add(subscriptionName.toString()); .setName(subscriptionName.toString())
log.info("Created new subscription: [{}]", subscriptionName.toString()); .setTopic(topicName.toString());
} catch (IOException e) {
log.error("Failed to create subscription: [{}].", subscriptionName.toString(), e); setAckDeadline(subscriptionBuilder);
throw new RuntimeException("Failed to create subscription.", e); setMessageRetention(subscriptionBuilder);
subscriptionAdminClient.createSubscription(subscriptionBuilder.build());
subscriptionSet.add(subscriptionName.toString());
log.info("Created new subscription: [{}]", subscriptionName.toString());
}
private void setAckDeadline(Subscription.Builder builder) {
if (subscriptionProperties.containsKey(ACK_DEADLINE)) {
builder.setAckDeadlineSeconds(Integer.parseInt(subscriptionProperties.get(ACK_DEADLINE)));
} }
} }
private void setMessageRetention(Subscription.Builder builder) {
if (subscriptionProperties.containsKey(MESSAGE_RETENTION)) {
Duration duration = Duration
.newBuilder()
.setSeconds(Long.parseLong(subscriptionProperties.get(MESSAGE_RETENTION)))
.build();
builder.setMessageRetentionDuration(duration);
}
}
@Override
public void destroy() {
if (topicAdminClient != null) {
topicAdminClient.close();
}
if (subscriptionAdminClient != null) {
subscriptionAdminClient.close();
}
}
} }

3
common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java

@ -40,9 +40,6 @@ public class TbPubSubSettings {
@Value("${queue.pubsub.service_account}") @Value("${queue.pubsub.service_account}")
private String serviceAccount; private String serviceAccount;
@Value("${queue.pubsub.ack_deadline}")
private int ackDeadline;
@Value("${queue.pubsub.max_msg_size}") @Value("${queue.pubsub.max_msg_size}")
private int maxMsgSize; private int maxMsgSize;

71
common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java

@ -0,0 +1,71 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.pubsub;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.HashMap;
import java.util.Map;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='pubsub'")
public class TbPubSubSubscriptionSettings {
@Value("${queue.pubsub.queue-properties.core}")
private String coreProperties;
@Value("${queue.pubsub.queue-properties.rule-engine}")
private String ruleEngineProperties;
@Value("${queue.pubsub.queue-properties.transport-api}")
private String transportApiProperties;
@Value("${queue.pubsub.queue-properties.notifications}")
private String notificationsProperties;
@Value("${queue.pubsub.queue-properties.js-executor}")
private String jsExecutorProperties;
@Getter
private Map<String, String> coreSettings;
@Getter
private Map<String, String> ruleEngineSettings;
@Getter
private Map<String, String> transportApiSettings;
@Getter
private Map<String, String> notificationsSettings;
@Getter
private Map<String, String> jsExecutorSettings;
@PostConstruct
private void init() {
coreSettings = getSettings(coreProperties);
ruleEngineSettings = getSettings(ruleEngineProperties);
transportApiSettings = getSettings(transportApiProperties);
notificationsSettings = getSettings(notificationsProperties);
jsExecutorSettings = getSettings(jsExecutorProperties);
}
private Map<String, String> getSettings(String properties) {
Map<String, String> configs = new HashMap<>();
for (String property : properties.split(";")) {
int delimiterPosition = property.indexOf(":");
String key = property.substring(0, delimiterPosition);
String value = property.substring(delimiterPosition + 1);
configs.put(key, value);
}
return configs;
}
}

11
common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java

@ -18,24 +18,23 @@ package org.thingsboard.server.queue.rabbitmq;
import com.rabbitmq.client.Channel; import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Connection; import com.rabbitmq.client.Connection;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Component;
import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin;
import java.io.IOException; import java.io.IOException;
import java.util.Map;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
@Slf4j @Slf4j
@Component
@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'")
public class TbRabbitMqAdmin implements TbQueueAdmin { public class TbRabbitMqAdmin implements TbQueueAdmin {
private final TbRabbitMqSettings rabbitMqSettings; private final TbRabbitMqSettings rabbitMqSettings;
private final Channel channel; private final Channel channel;
private final Connection connection; private final Connection connection;
private final Map<String, Object> arguments;
public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings) { public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings, Map<String, Object> arguments) {
this.rabbitMqSettings = rabbitMqSettings; this.rabbitMqSettings = rabbitMqSettings;
this.arguments = arguments;
try { try {
connection = rabbitMqSettings.getConnectionFactory().newConnection(); connection = rabbitMqSettings.getConnectionFactory().newConnection();
@ -55,7 +54,7 @@ public class TbRabbitMqAdmin implements TbQueueAdmin {
@Override @Override
public void createTopicIfNotExists(String topic) { public void createTopicIfNotExists(String topic) {
try { try {
channel.queueDeclare(topic, false, false, false, null); channel.queueDeclare(topic, false, false, false, arguments);
} catch (IOException e) { } catch (IOException e) {
log.error("Failed to bind queue: [{}]", topic, e); log.error("Failed to bind queue: [{}]", topic, e);
} }

98
common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java

@ -0,0 +1,98 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.rabbitmq;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'")
public class TbRabbitMqQueueArguments {
@Value("${queue.rabbitmq.queue-properties.core}")
private String coreProperties;
@Value("${queue.rabbitmq.queue-properties.rule-engine}")
private String ruleEngineProperties;
@Value("${queue.rabbitmq.queue-properties.transport-api}")
private String transportApiProperties;
@Value("${queue.rabbitmq.queue-properties.notifications}")
private String notificationsProperties;
@Value("${queue.rabbitmq.queue-properties.js-executor}")
private String jsExecutorProperties;
@Getter
private Map<String, Object> coreArgs;
@Getter
private Map<String, Object> ruleEngineArgs;
@Getter
private Map<String, Object> transportApiArgs;
@Getter
private Map<String, Object> notificationsArgs;
@Getter
private Map<String, Object> jsExecutorArgs;
@PostConstruct
private void init() {
coreArgs = getArgs(coreProperties);
ruleEngineArgs = getArgs(ruleEngineProperties);
transportApiArgs = getArgs(transportApiProperties);
notificationsArgs = getArgs(notificationsProperties);
jsExecutorArgs = getArgs(jsExecutorProperties);
}
private Map<String, Object> getArgs(String properties) {
Map<String, Object> configs = new HashMap<>();
for (String property : properties.split(";")) {
int delimiterPosition = property.indexOf(":");
String key = property.substring(0, delimiterPosition);
String strValue = property.substring(delimiterPosition + 1);
configs.put(key, getObjectValue(strValue));
}
return configs;
}
private Object getObjectValue(String str) {
if (str.equalsIgnoreCase("true") || str.equalsIgnoreCase("false")) {
return Boolean.valueOf(str);
} else if (isNumeric(str)) {
return getNumericValue(str);
}
return str;
}
private Object getNumericValue(String str) {
if (str.contains(".")) {
return Double.valueOf(str);
} else {
return Long.valueOf(str);
}
}
private static final Pattern PATTERN = Pattern.compile("-?\\d+(\\.\\d+)?");
public boolean isNumeric(String strNum) {
if (strNum == null) {
return false;
}
return PATTERN.matcher(strNum).matches();
}
}

2
common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java

@ -24,7 +24,7 @@ public class TbRuleEngineQueueConfiguration {
private String topic; private String topic;
private int pollInterval; private int pollInterval;
private int partitions; private int partitions;
private String packProcessingTimeout; private long packProcessingTimeout;
private TbRuleEngineQueueSubmitStrategyConfiguration submitStrategy; private TbRuleEngineQueueSubmitStrategyConfiguration submitStrategy;
private TbRuleEngineQueueAckStrategyConfiguration processingStrategy; private TbRuleEngineQueueAckStrategyConfiguration processingStrategy;

12
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java

@ -138,12 +138,24 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
break; break;
case PUBLISH: case PUBLISH:
processPublish(ctx, (MqttPublishMessage) msg); processPublish(ctx, (MqttPublishMessage) msg);
transportService.reportActivity(sessionInfo);
if (gatewaySessionHandler != null) {
gatewaySessionHandler.reportActivity();
}
break; break;
case SUBSCRIBE: case SUBSCRIBE:
processSubscribe(ctx, (MqttSubscribeMessage) msg); processSubscribe(ctx, (MqttSubscribeMessage) msg);
transportService.reportActivity(sessionInfo);
if (gatewaySessionHandler != null) {
gatewaySessionHandler.reportActivity();
}
break; break;
case UNSUBSCRIBE: case UNSUBSCRIBE:
processUnsubscribe(ctx, (MqttUnsubscribeMessage) msg); processUnsubscribe(ctx, (MqttUnsubscribeMessage) msg);
transportService.reportActivity(sessionInfo);
if (gatewaySessionHandler != null) {
gatewaySessionHandler.reportActivity();
}
break; break;
case PINGREQ: case PINGREQ:
if (checkConnected(ctx, msg)) { if (checkConnected(ctx, msg)) {

8
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java

@ -544,6 +544,9 @@ public class DefaultTransportService implements TransportService {
protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback<Void> callback) { protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback<Void> callback) {
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, getTenantId(sessionInfo), getDeviceId(sessionInfo)); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, getTenantId(sessionInfo), getDeviceId(sessionInfo));
if (log.isTraceEnabled()) {
log.trace("[{}][{}] Pushing to topic {} message {}", getTenantId(sessionInfo), getDeviceId(sessionInfo), tpi.getFullTopicName(), toDeviceActorMsg);
}
tbCoreMsgProducer.send(tpi, tbCoreMsgProducer.send(tpi,
new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), new TbProtoQueueMsg<>(getRoutingKey(sessionInfo),
ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ? ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ?
@ -552,6 +555,9 @@ public class DefaultTransportService implements TransportService {
protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) { protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) {
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator());
if (log.isTraceEnabled()) {
log.trace("[{}][{}] Pushing to topic {} message {}", tenantId, tbMsg.getOriginator(), tpi.getFullTopicName(), tbMsg);
}
ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg)) ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg))
.setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build();
@ -588,7 +594,7 @@ public class DefaultTransportService implements TransportService {
@Override @Override
public void onSuccess(TbQueueMsgMetadata metadata) { public void onSuccess(TbQueueMsgMetadata metadata) {
if (msgCount.decrementAndGet() <= 0) { if (msgCount.decrementAndGet() <= 0) {
callback.onSuccess(null); DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onSuccess(null));
} }
} }

3
dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java

@ -32,6 +32,9 @@ public class ModelConstants {
public static final String NULL_UUID_STR = UUIDConverter.fromTimeUUID(NULL_UUID); public static final String NULL_UUID_STR = UUIDConverter.fromTimeUUID(NULL_UUID);
public static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); public static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID);
// this is the difference between midnight October 15, 1582 UTC and midnight January 1, 1970 UTC as 100 nanosecond units
public static final long EPOCH_DIFF = 122192928000000000L;
/** /**
* Generic constants. * Generic constants.
*/ */

14
dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java

@ -37,6 +37,9 @@ import javax.persistence.EnumType;
import javax.persistence.Enumerated; import javax.persistence.Enumerated;
import javax.persistence.Table; import javax.persistence.Table;
import java.util.UUID;
import static org.thingsboard.server.dao.model.ModelConstants.EPOCH_DIFF;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_BODY_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_BODY_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_COLUMN_FAMILY_NAME; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_COLUMN_FAMILY_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_PROPERTY;
@ -44,6 +47,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TENANT_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TENANT_ID_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_UID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_UID_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN;
@Data @Data
@EqualsAndHashCode(callSuper = true) @EqualsAndHashCode(callSuper = true)
@ -73,9 +77,15 @@ public class EventEntity extends BaseSqlEntity<Event> implements BaseEntity<Eve
@Column(name = EVENT_BODY_PROPERTY) @Column(name = EVENT_BODY_PROPERTY)
private JsonNode body; private JsonNode body;
@Column(name = TS_COLUMN)
private long ts;
public EventEntity(Event event) { public EventEntity(Event event) {
if (event.getId() != null) { if (event.getId() != null) {
this.setUuid(event.getId().getId()); this.setUuid(event.getId().getId());
this.ts = getTs(event.getId().getId());
} else {
this.ts = System.currentTimeMillis();
} }
if (event.getTenantId() != null) { if (event.getTenantId() != null) {
this.tenantId = toString(event.getTenantId().getId()); this.tenantId = toString(event.getTenantId().getId());
@ -101,4 +111,8 @@ public class EventEntity extends BaseSqlEntity<Event> implements BaseEntity<Eve
event.setUid(eventUid); event.setUid(eventUid);
return event; return event;
} }
private long getTs(UUID uuid) {
return (uuid.timestamp() - EPOCH_DIFF) / 10000;
}
} }

25
dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java

@ -88,26 +88,33 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC
RuleChain ruleChain = ruleChainDao.findById(tenantId, ruleChainId.getId()); RuleChain ruleChain = ruleChainDao.findById(tenantId, ruleChainId.getId());
if (!ruleChain.isRoot()) { if (!ruleChain.isRoot()) {
RuleChain previousRootRuleChain = getRootTenantRuleChain(ruleChain.getTenantId()); RuleChain previousRootRuleChain = getRootTenantRuleChain(ruleChain.getTenantId());
if (!previousRootRuleChain.getId().equals(ruleChain.getId())) { try {
try { if (previousRootRuleChain == null) {
setRootAndSave(tenantId, ruleChain);
return true;
} else if (!previousRootRuleChain.getId().equals(ruleChain.getId())) {
deleteRelation(tenantId, new EntityRelation(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), deleteRelation(tenantId, new EntityRelation(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(),
EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN)); EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN));
previousRootRuleChain.setRoot(false); previousRootRuleChain.setRoot(false);
ruleChainDao.save(tenantId, previousRootRuleChain); ruleChainDao.save(tenantId, previousRootRuleChain);
createRelation(tenantId, new EntityRelation(ruleChain.getTenantId(), ruleChain.getId(), setRootAndSave(tenantId, ruleChain);
EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN));
ruleChain.setRoot(true);
ruleChainDao.save(tenantId, ruleChain);
return true; return true;
} catch (ExecutionException | InterruptedException e) {
log.warn("[{}] Failed to set root rule chain, ruleChainId: [{}]", ruleChainId);
throw new RuntimeException(e);
} }
} catch (ExecutionException | InterruptedException e) {
log.warn("[{}] Failed to set root rule chain, ruleChainId: [{}]", ruleChainId);
throw new RuntimeException(e);
} }
} }
return false; return false;
} }
private void setRootAndSave(TenantId tenantId, RuleChain ruleChain) throws ExecutionException, InterruptedException {
createRelation(tenantId, new EntityRelation(ruleChain.getTenantId(), ruleChain.getId(),
EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN));
ruleChain.setRoot(true);
ruleChainDao.save(tenantId, ruleChain);
}
@Override @Override
public RuleChainMetaData saveRuleChainMetaData(TenantId tenantId, RuleChainMetaData ruleChainMetaData) { public RuleChainMetaData saveRuleChainMetaData(TenantId tenantId, RuleChainMetaData ruleChainMetaData) {
Validator.validateId(ruleChainMetaData.getRuleChainId(), "Incorrect rule chain id."); Validator.validateId(ruleChainMetaData.getRuleChainId(), "Incorrect rule chain id.");

3
dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java

@ -75,7 +75,8 @@ public abstract class AbstractEventInsertRepository implements EventInsertReposi
.setParameter("entity_type", entity.getEntityType().name()) .setParameter("entity_type", entity.getEntityType().name())
.setParameter("event_type", entity.getEventType()) .setParameter("event_type", entity.getEventType())
.setParameter("event_uid", entity.getEventUid()) .setParameter("event_uid", entity.getEventUid())
.setParameter("tenant_id", entity.getTenantId()); .setParameter("tenant_id", entity.getTenantId())
.setParameter("ts", entity.getTs());
} }
private EventEntity processSaveOrUpdate(EventEntity entity, String query) { private EventEntity processSaveOrUpdate(EventEntity entity, String query) {

4
dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java

@ -44,7 +44,7 @@ public class HsqlEventInsertRepository extends AbstractEventInsertRepository {
} }
private static String getInsertString(String conflictStatement) { private static String getInsertString(String conflictStatement) {
return "MERGE INTO event USING (VALUES :id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id) I (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) ON " + conflictStatement + " WHEN MATCHED THEN UPDATE SET event.id = I.id, event.body = I.body, event.entity_id = I.entity_id, event.entity_type = I.entity_type, event.event_type = I.event_type, event.event_uid = I.event_uid, event.tenant_id = I.tenant_id" + return "MERGE INTO event USING (VALUES :id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id, :ts) I (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) ON " + conflictStatement + " WHEN MATCHED THEN UPDATE SET event.id = I.id, event.body = I.body, event.entity_id = I.entity_id, event.entity_type = I.entity_type, event.event_type = I.event_type, event.event_uid = I.event_uid, event.tenant_id = I.tenant_id, event.ts = I.ts" +
" WHEN NOT MATCHED THEN INSERT (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) VALUES (I.id, I.body, I.entity_id, I.entity_type, I.event_type, I.event_uid, I.tenant_id)"; " WHEN NOT MATCHED THEN INSERT (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) VALUES (I.id, I.body, I.entity_id, I.entity_type, I.event_type, I.event_uid, I.tenant_id, I.ts)";
} }
} }

2
dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java

@ -48,6 +48,6 @@ public class PsqlEventInsertRepository extends AbstractEventInsertRepository {
} }
private static String getInsertOrUpdateString(String eventKeyStatement, String updateKeyStatement) { private static String getInsertOrUpdateString(String eventKeyStatement, String updateKeyStatement) {
return "INSERT INTO event (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) VALUES (:id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id) ON CONFLICT " + eventKeyStatement + " DO UPDATE SET body = :body, " + updateKeyStatement + " returning *"; return "INSERT INTO event (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) VALUES (:id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id, :ts) ON CONFLICT " + eventKeyStatement + " DO UPDATE SET body = :body, ts = :ts," + updateKeyStatement + " returning *";
} }
} }

2
dao/src/main/resources/sql/schema-entities-hsql.sql

@ -144,6 +144,7 @@ CREATE TABLE IF NOT EXISTS event (
event_type varchar(255), event_type varchar(255),
event_uid varchar(255), event_uid varchar(255),
tenant_id varchar(31), tenant_id varchar(31),
ts bigint NOT NULL,
CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid) CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid)
); );
@ -251,3 +252,4 @@ CREATE TABLE IF NOT EXISTS entity_view (
search_text varchar(255), search_text varchar(255),
additional_info varchar additional_info varchar
); );

26
dao/src/main/resources/sql/schema-entities.sql

@ -144,6 +144,7 @@ CREATE TABLE IF NOT EXISTS event (
event_type varchar(255), event_type varchar(255),
event_uid varchar(255), event_uid varchar(255),
tenant_id varchar(31), tenant_id varchar(31),
ts bigint NOT NULL,
CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid) CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid)
); );
@ -251,3 +252,28 @@ CREATE TABLE IF NOT EXISTS entity_view (
search_text varchar(255), search_text varchar(255),
additional_info varchar additional_info varchar
); );
CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint)
LANGUAGE plpgsql AS
$$
DECLARE
ttl_ts bigint;
debug_ttl_ts bigint;
ttl_deleted_count bigint DEFAULT 0;
debug_ttl_deleted_count bigint DEFAULT 0;
BEGIN
IF ttl > 0 THEN
ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint;
EXECUTE format(
'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count;
END IF;
IF debug_ttl > 0 THEN
debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint;
EXECUTE format(
'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count;
END IF;
RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count;
RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count;
deleted := ttl_deleted_count + debug_ttl_deleted_count;
END
$$;

2
dao/src/main/resources/sql/schema-timescale.sql

@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings
CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version)
); );
INSERT INTO tb_schema_settings (schema_version) VALUES (2005000); INSERT INTO tb_schema_settings (schema_version) VALUES (2005000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005000;
CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS
$$ $$

2
dao/src/main/resources/sql/schema-ts-psql.sql

@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings
CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version)
); );
INSERT INTO tb_schema_settings (schema_version) VALUES (2005000); INSERT INTO tb_schema_settings (schema_version) VALUES (2005000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005000;
CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint) CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint)
LANGUAGE plpgsql AS LANGUAGE plpgsql AS

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save