tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

merge with master

pull/2685/head
YevhenBondarenko 7 years ago
parent
0dc31fbdde cb6e91277f
commit
6322da3d04
151 changed files with 5585 additions and 688 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      application/src/main/conf/logback.xml
  2. 1274
      application/src/main/data/json/demo/dashboards/gateways.json
  3. 520
      application/src/main/data/json/demo/dashboards/rule_engine_statistics.json
  4. 1236
      application/src/main/data/json/demo/dashboards/theromstats.json
  5. 146
      application/src/main/data/json/demo/rule_chains/root_rule_chain.json
  6. 137
      application/src/main/data/json/demo/rule_chains/thermostat_alarms.json
  7. 32
      application/src/main/data/json/system/widget_bundles/gateway_widgets.json
  8. 25
      application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql
  9. 3
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  10. 3
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
  11. 5
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java
  12. 2
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java
  13. 40
      application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java
  14. 2
      application/src/main/java/org/thingsboard/server/controller/QueueController.java
  15. 14
      application/src/main/java/org/thingsboard/server/controller/RuleChainController.java
  16. 11
      application/src/main/java/org/thingsboard/server/controller/TelemetryController.java
  17. 2
      application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
  18. 34
      application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java
  19. 27
      application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java
  20. 5
      application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java
  21. 6
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java
  22. 15
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  23. 10
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
  24. 2
      application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java
  25. 8
      application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java
  26. 25
      application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java
  27. 28
      application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java
  28. 90
      application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java
  29. 11
      application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java
  30. 6
      application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java
  31. 16
      application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java
  32. 4
      application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java
  33. 10
      application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java
  34. 8
      application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java
  35. 2
      application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java
  36. 11
      application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java
  37. 8
      application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java
  38. 40
      application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java
  39. 59
      application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java
  40. 49
      application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java
  41. 2
      application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java
  42. 2
      application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java
  43. 3
      application/src/main/resources/logback.xml
  44. 107
      application/src/main/resources/thingsboard.yml
  45. 128
      application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java
  46. 6
      application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java
  47. 7
      application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java
  48. 7
      application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java
  49. 7
      application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java
  50. 7
      application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java
  51. 19
      application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java
  52. 11
      application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java
  53. 7
      application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java
  54. 7
      application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java
  55. 4
      application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java
  56. 21
      application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java
  57. 66
      application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java
  58. 7
      application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java
  59. 7
      application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java
  60. 38
      common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java
  61. 71
      common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java
  62. 12
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java
  63. 42
      common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java
  64. 11
      common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java
  65. 11
      common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java
  66. 9
      common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java
  67. 20
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java
  68. 19
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java
  69. 17
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java
  70. 18
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java
  71. 64
      common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java
  72. 54
      common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java
  73. 50
      common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java
  74. 54
      common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java
  75. 71
      common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java
  76. 57
      common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java
  77. 48
      common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java
  78. 50
      common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java
  79. 63
      common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java
  80. 57
      common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java
  81. 48
      common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java
  82. 51
      common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java
  83. 120
      common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java
  84. 3
      common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java
  85. 71
      common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java
  86. 11
      common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java
  87. 98
      common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java
  88. 2
      common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java
  89. 12
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java
  90. 8
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java
  91. 3
      dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java
  92. 14
      dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java
  93. 25
      dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java
  94. 3
      dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java
  95. 4
      dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java
  96. 2
      dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java
  97. 2
      dao/src/main/resources/sql/schema-entities-hsql.sql
  98. 26
      dao/src/main/resources/sql/schema-entities.sql
  99. 2
      dao/src/main/resources/sql/schema-timescale.sql
  100. 2
      dao/src/main/resources/sql/schema-ts-psql.sql

1
application/src/main/conf/logback.xml
View File

@ -36,6 +36,7 @@
<logger name="org.thingsboard.server" level="INFO" />
<logger name="akka" level="INFO" />
<logger name="com.microsoft.azure.servicebus.primitives.CoreMessageReceiver" level="OFF" />
<root level="INFO">
<appender-ref ref="fileLogAppender"/>

1274
application/src/main/data/json/demo/dashboards/gateways.json
View File

File diff suppressed because it is too large

520
application/src/main/data/json/demo/dashboards/rule_engine_statistics.json
View File

@ -0,0 +1,520 @@
{
"title": "Rule Engine Statistics",
"configuration": {
"widgets": {
"81987f19-3eac-e4ce-b790-d96e9b54d9a0": {
"isSystemType": true,
"bundleAlias": "charts",
"typeAlias": "basic_timeseries",
"type": "timeseries",
"title": "New widget",
"sizeX": 12,
"sizeY": 7,
"config": {
"datasources": [
{
"type": "entity",
"dataKeys": [
{
"name": "successfulMsgs",
"type": "timeseries",
"label": "${entityName} Successful",
"color": "#4caf50",
"settings": {
"excludeFromStacking": false,
"hideDataByDefault": false,
"disableDataHiding": false,
"removeFromLegend": false,
"showLines": true,
"fillLines": false,
"showPoints": false,
"showPointShape": "circle",
"pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);",
"showPointsLineWidth": 5,
"showPointsRadius": 3,
"showSeparateAxis": false,
"axisPosition": "left",
"thresholds": [
{
"thresholdValueSource": "predefinedValue"
}
],
"comparisonSettings": {
"showValuesForComparison": true
}
},
"_hash": 0.15490750967648736
},
{
"name": "failedMsgs",
"type": "timeseries",
"label": "${entityName} Permanent Failures",
"color": "#ef5350",
"settings": {
"excludeFromStacking": false,
"hideDataByDefault": false,
"disableDataHiding": false,
"removeFromLegend": false,
"showLines": true,
"fillLines": false,
"showPoints": false,
"showPointShape": "circle",
"pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);",
"showPointsLineWidth": 5,
"showPointsRadius": 3,
"showSeparateAxis": false,
"axisPosition": "left",
"thresholds": [
{
"thresholdValueSource": "predefinedValue"
}
],
"comparisonSettings": {
"showValuesForComparison": true
}
},
"_hash": 0.4186621166514697
},
{
"name": "tmpFailed",
"type": "timeseries",
"label": "${entityName} Processing Failures",
"color": "#ffc107",
"settings": {
"excludeFromStacking": false,
"hideDataByDefault": false,
"disableDataHiding": false,
"removeFromLegend": false,
"showLines": true,
"fillLines": false,
"showPoints": false,
"showPointShape": "circle",
"pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);",
"showPointsLineWidth": 5,
"showPointsRadius": 3,
"showSeparateAxis": false,
"axisPosition": "left",
"thresholds": [
{
"thresholdValueSource": "predefinedValue"
}
],
"comparisonSettings": {
"showValuesForComparison": true
}
},
"_hash": 0.49891007198715376
}
],
"entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018"
}
],
"timewindow": {
"realtime": {
"interval": 1000,
"timewindowMs": 300000
},
"aggregation": {
"type": "NONE",
"limit": 8640
},
"hideInterval": false,
"hideAggregation": false,
"hideAggInterval": false
},
"showTitle": true,
"backgroundColor": "#fff",
"color": "rgba(0, 0, 0, 0.87)",
"padding": "8px",
"settings": {
"shadowSize": 4,
"fontColor": "#545454",
"fontSize": 10,
"xaxis": {
"showLabels": true,
"color": "#545454"
},
"yaxis": {
"showLabels": true,
"color": "#545454"
},
"grid": {
"color": "#545454",
"tickColor": "#DDDDDD",
"verticalLines": true,
"horizontalLines": true,
"outlineWidth": 1
},
"stack": false,
"tooltipIndividual": false,
"timeForComparison": "months",
"xaxisSecond": {
"axisPosition": "top",
"showLabels": true
}
},
"title": "Queue Stats",
"dropShadow": true,
"enableFullscreen": true,
"titleStyle": {
"fontSize": "16px",
"fontWeight": 400
},
"mobileHeight": null,
"showTitleIcon": false,
"titleIcon": null,
"iconColor": "rgba(0, 0, 0, 0.87)",
"iconSize": "24px",
"titleTooltip": "",
"widgetStyle": {},
"useDashboardTimewindow": false,
"displayTimewindow": true,
"showLegend": true,
"actions": {},
"legendConfig": {
"direction": "column",
"position": "bottom",
"showMin": true,
"showMax": true,
"showAvg": false,
"showTotal": true
}
},
"id": "81987f19-3eac-e4ce-b790-d96e9b54d9a0"
},
"5eb79712-5c24-3060-7e4f-6af36b8f842d": {
"isSystemType": true,
"bundleAlias": "cards",
"typeAlias": "timeseries_table",
"type": "timeseries",
"title": "New widget",
"sizeX": 24,
"sizeY": 5,
"config": {
"datasources": [
{
"type": "entity",
"dataKeys": [
{
"name": "ruleEngineException",
"type": "timeseries",
"label": "Rule Chain",
"color": "#2196f3",
"settings": {
"useCellStyleFunction": false,
"useCellContentFunction": true,
"cellContentFunction": "return JSON.parse(value).ruleChainName;"
},
"_hash": 0.9954481282345906
},
{
"name": "ruleEngineException",
"type": "timeseries",
"label": "Rule Node",
"color": "#4caf50",
"settings": {
"useCellStyleFunction": false,
"useCellContentFunction": true,
"cellContentFunction": "return JSON.parse(value).ruleNodeName;"
},
"_hash": 0.18580357036589978
},
{
"name": "ruleEngineException",
"type": "timeseries",
"label": "Latest Error",
"color": "#f44336",
"settings": {
"useCellStyleFunction": false,
"useCellContentFunction": true,
"cellContentFunction": "return JSON.parse(value).message;"
},
"_hash": 0.7255162989552142
}
],
"entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018"
}
],
"timewindow": {
"realtime": {
"interval": 1000,
"timewindowMs": 86400000
},
"aggregation": {
"type": "NONE",
"limit": 200
}
},
"showTitle": true,
"backgroundColor": "rgb(255, 255, 255)",
"color": "rgba(0, 0, 0, 0.87)",
"padding": "8px",
"settings": {
"showTimestamp": true,
"displayPagination": true,
"defaultPageSize": 10
},
"title": "Exceptions",
"dropShadow": true,
"enableFullscreen": true,
"titleStyle": {
"fontSize": "16px",
"fontWeight": 400
},
"useDashboardTimewindow": false,
"showLegend": false,
"widgetStyle": {},
"actions": {},
"showTitleIcon": false,
"titleIcon": null,
"iconColor": "rgba(0, 0, 0, 0.87)",
"iconSize": "24px",
"titleTooltip": "",
"displayTimewindow": true
},
"id": "5eb79712-5c24-3060-7e4f-6af36b8f842d"
},
"ad3f1417-87a8-750e-fc67-49a2de1466d4": {
"isSystemType": true,
"bundleAlias": "charts",
"typeAlias": "basic_timeseries",
"type": "timeseries",
"title": "New widget",
"sizeX": 12,
"sizeY": 7,
"config": {
"datasources": [
{
"type": "entity",
"dataKeys": [
{
"name": "timeoutMsgs",
"type": "timeseries",
"label": "${entityName} Permanent Timeouts",
"color": "#4caf50",
"settings": {
"excludeFromStacking": false,
"hideDataByDefault": false,
"disableDataHiding": false,
"removeFromLegend": false,
"showLines": true,
"fillLines": false,
"showPoints": false,
"showPointShape": "circle",
"pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);",
"showPointsLineWidth": 5,
"showPointsRadius": 3,
"showSeparateAxis": false,
"axisPosition": "left",
"thresholds": [
{
"thresholdValueSource": "predefinedValue"
}
],
"comparisonSettings": {
"showValuesForComparison": true
}
},
"_hash": 0.565222981550328
},
{
"name": "tmpTimeout",
"type": "timeseries",
"label": "${entityName} Processing Timeouts",
"color": "#9c27b0",
"settings": {
"excludeFromStacking": false,
"hideDataByDefault": false,
"disableDataHiding": false,
"removeFromLegend": false,
"showLines": true,
"fillLines": false,
"showPoints": false,
"showPointShape": "circle",
"pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);",
"showPointsLineWidth": 5,
"showPointsRadius": 3,
"showSeparateAxis": false,
"axisPosition": "left",
"thresholds": [
{
"thresholdValueSource": "predefinedValue"
}
],
"comparisonSettings": {
"showValuesForComparison": true
}
},
"_hash": 0.2679547062508352
}
],
"entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018"
}
],
"timewindow": {
"realtime": {
"interval": 1000,
"timewindowMs": 300000
},
"aggregation": {
"type": "NONE",
"limit": 8640
},
"hideInterval": false,
"hideAggregation": false,
"hideAggInterval": false
},
"showTitle": true,
"backgroundColor": "#fff",
"color": "rgba(0, 0, 0, 0.87)",
"padding": "8px",
"settings": {
"shadowSize": 4,
"fontColor": "#545454",
"fontSize": 10,
"xaxis": {
"showLabels": true,
"color": "#545454"
},
"yaxis": {
"showLabels": true,
"color": "#545454"
},
"grid": {
"color": "#545454",
"tickColor": "#DDDDDD",
"verticalLines": true,
"horizontalLines": true,
"outlineWidth": 1
},
"stack": false,
"tooltipIndividual": false,
"timeForComparison": "months",
"xaxisSecond": {
"axisPosition": "top",
"showLabels": true
}
},
"title": "Processing Failures and Timeouts",
"dropShadow": true,
"enableFullscreen": true,
"titleStyle": {
"fontSize": "16px",
"fontWeight": 400
},
"mobileHeight": null,
"showTitleIcon": false,
"titleIcon": null,
"iconColor": "rgba(0, 0, 0, 0.87)",
"iconSize": "24px",
"titleTooltip": "",
"widgetStyle": {},
"useDashboardTimewindow": false,
"displayTimewindow": true,
"showLegend": true,
"actions": {},
"legendConfig": {
"direction": "column",
"position": "bottom",
"showMin": true,
"showMax": true,
"showAvg": false,
"showTotal": true
}
},
"id": "ad3f1417-87a8-750e-fc67-49a2de1466d4"
}
},
"states": {
"default": {
"name": "Rule Engine Statistics",
"root": true,
"layouts": {
"main": {
"widgets": {
"81987f19-3eac-e4ce-b790-d96e9b54d9a0": {
"sizeX": 12,
"sizeY": 7,
"mobileHeight": null,
"row": 0,
"col": 0
},
"5eb79712-5c24-3060-7e4f-6af36b8f842d": {
"sizeX": 24,
"sizeY": 5,
"row": 7,
"col": 0
},
"ad3f1417-87a8-750e-fc67-49a2de1466d4": {
"sizeX": 12,
"sizeY": 7,
"mobileHeight": null,
"row": 0,
"col": 12
}
},
"gridSettings": {
"backgroundColor": "#eeeeee",
"color": "rgba(0,0,0,0.870588)",
"columns": 24,
"margins": [
10,
10
],
"backgroundSizeMode": "100%",
"autoFillHeight": true,
"mobileAutoFillHeight": false,
"mobileRowHeight": 70
}
}
}
}
},
"entityAliases": {
"140f23dd-e3a0-ed98-6189-03c49d2d8018": {
"id": "140f23dd-e3a0-ed98-6189-03c49d2d8018",
"alias": "TbServiceQueues",
"filter": {
"type": "assetType",
"resolveMultiple": true,
"assetType": "TbServiceQueue",
"assetNameFilter": ""
}
}
},
"timewindow": {
"displayValue": "",
"selectedTab": 0,
"hideInterval": false,
"hideAggregation": false,
"hideAggInterval": false,
"realtime": {
"interval": 1000,
"timewindowMs": 60000
},
"history": {
"historyType": 0,
"interval": 1000,
"timewindowMs": 60000,
"fixedTimewindow": {
"startTimeMs": 1586176634823,
"endTimeMs": 1586263034823
}
},
"aggregation": {
"type": "AVG",
"limit": 25000
}
},
"settings": {
"stateControllerId": "entity",
"showTitle": false,
"showDashboardsSelect": true,
"showEntitiesSelect": true,
"showDashboardTimewindow": true,
"showDashboardExport": true,
"toolbarAlwaysOpen": true
}
},
"name": "Rule Engine Statistics"
}

1236
application/src/main/data/json/demo/dashboards/theromstats.json
View File

File diff suppressed because it is too large

146
application/src/main/data/json/demo/rule_chains/root_rule_chain.json
View File

@ -0,0 +1,146 @@
{
"ruleChain": {
"additionalInfo": null,
"name": "Root Rule Chain",
"firstRuleNodeId": null,
"root": true,
"debugMode": false,
"configuration": null
},
"metadata": {
"firstNodeIndex": 2,
"nodes": [
{
"additionalInfo": {
"layoutX": 824,
"layoutY": 156
},
"type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode",
"name": "Save Timeseries",
"debugMode": false,
"configuration": {
"defaultTTL": 0
}
},
{
"additionalInfo": {
"layoutX": 825,
"layoutY": 52
},
"type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode",
"name": "Save Client Attributes",
"debugMode": false,
"configuration": {
"scope": "CLIENT_SCOPE"
}
},
{
"additionalInfo": {
"layoutX": 347,
"layoutY": 149
},
"type": "org.thingsboard.rule.engine.filter.TbMsgTypeSwitchNode",
"name": "Message Type Switch",
"debugMode": false,
"configuration": {
"version": 0
}
},
{
"additionalInfo": {
"layoutX": 825,
"layoutY": 266
},
"type": "org.thingsboard.rule.engine.action.TbLogNode",
"name": "Log RPC from Device",
"debugMode": false,
"configuration": {
"jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);"
}
},
{
"additionalInfo": {
"layoutX": 825,
"layoutY": 379
},
"type": "org.thingsboard.rule.engine.action.TbLogNode",
"name": "Log Other",
"debugMode": false,
"configuration": {
"jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);"
}
},
{
"additionalInfo": {
"layoutX": 825,
"layoutY": 468
},
"type": "org.thingsboard.rule.engine.rpc.TbSendRPCRequestNode",
"name": "RPC Call Request",
"debugMode": false,
"configuration": {
"timeoutInSeconds": 60
}
},
{
"additionalInfo": {
"layoutX": 1069,
"layoutY": 90
},
"type": "org.thingsboard.rule.engine.filter.TbJsFilterNode",
"name": "Is Thermostat?",
"debugMode": false,
"configuration": {
"jsScript": "return metadata[\"deviceType\"] === \"thermostat\";"
}
}
],
"connections": [
{
"fromIndex": 0,
"toIndex": 6,
"type": "Success"
},
{
"fromIndex": 2,
"toIndex": 4,
"type": "Other"
},
{
"fromIndex": 2,
"toIndex": 1,
"type": "Post attributes"
},
{
"fromIndex": 2,
"toIndex": 0,
"type": "Post telemetry"
},
{
"fromIndex": 2,
"toIndex": 3,
"type": "RPC Request from Device"
},
{
"fromIndex": 2,
"toIndex": 5,
"type": "RPC Request to Device"
}
],
"ruleChainConnections": [
{
"fromIndex": 6,
"targetRuleChainId": {
"entityType": "RULE_CHAIN",
"id": "83d42540-85fd-11ea-aee2-794850541ced"
},
"additionalInfo": {
"layoutX": 1088,
"layoutY": 203,
"ruleChainNodeId": "rule-chain-node-9"
},
"type": "True"
}
]
}
}

137
application/src/main/data/json/demo/rule_chains/thermostat_alarms.json
View File

@ -0,0 +1,137 @@
{
"ruleChain": {
"additionalInfo": null,
"name": "Thermostat Alarms",
"firstRuleNodeId": null,
"root": false,
"debugMode": false,
"configuration": null
},
"metadata": {
"firstNodeIndex": 5,
"nodes": [
{
"additionalInfo": {
"layoutX": 929,
"layoutY": 67
},
"type": "org.thingsboard.rule.engine.action.TbCreateAlarmNode",
"name": "Create Temp Alarm",
"debugMode": false,
"configuration": {
"alarmType": "High Temperature",
"alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\ndetails.triggerValue = msg.temperature;\nreturn details;",
"severity": "MAJOR",
"propagate": true,
"useMessageAlarmData": false,
"relationTypes": []
}
},
{
"additionalInfo": {
"layoutX": 930,
"layoutY": 201
},
"type": "org.thingsboard.rule.engine.action.TbClearAlarmNode",
"name": "Clear Temp Alarm",
"debugMode": false,
"configuration": {
"alarmType": "High Temperature",
"alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\nreturn details;"
}
},
{
"additionalInfo": {
"layoutX": 930,
"layoutY": 131
},
"type": "org.thingsboard.rule.engine.action.TbCreateAlarmNode",
"name": "Create Humidity Alarm",
"debugMode": false,
"configuration": {
"alarmType": "Low Humidity",
"alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\ndetails.triggerValue = msg.humidity;\nreturn details;",
"severity": "MINOR",
"propagate": true,
"useMessageAlarmData": false,
"relationTypes": []
}
},
{
"additionalInfo": {
"layoutX": 929,
"layoutY": 275
},
"type": "org.thingsboard.rule.engine.action.TbClearAlarmNode",
"name": "Clear Humidity Alarm",
"debugMode": false,
"configuration": {
"alarmType": "Low Humidity",
"alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\nreturn details;"
}
},
{
"additionalInfo": {
"layoutX": 586,
"layoutY": 148
},
"type": "org.thingsboard.rule.engine.filter.TbJsSwitchNode",
"name": "Check Alarms",
"debugMode": true,
"configuration": {
"jsScript": "var relations = [];\nif(metadata[\"ss_alarmTemperature\"] === \"true\"){\n if(msg.temperature > metadata[\"ss_thresholdTemperature\"]){\n relations.push(\"NewTempAlarm\");\n } else {\n relations.push(\"ClearTempAlarm\");\n }\n}\nif(metadata[\"ss_alarmHumidity\"] === \"true\"){\n if(msg.humidity < metadata[\"ss_thresholdHumidity\"]){\n relations.push(\"NewHumidityAlarm\");\n } else {\n relations.push(\"ClearHumidityAlarm\");\n }\n}\n\nreturn relations;"
}
},
{
"additionalInfo": {
"layoutX": 321,
"layoutY": 149
},
"type": "org.thingsboard.rule.engine.metadata.TbGetAttributesNode",
"name": "Fetch Configuration",
"debugMode": true,
"configuration": {
"clientAttributeNames": [],
"sharedAttributeNames": [],
"serverAttributeNames": [
"alarmTemperature",
"thresholdTemperature",
"alarmHumidity",
"thresholdHumidity"
],
"latestTsKeyNames": [],
"tellFailureIfAbsent": false,
"getLatestValueWithTs": false
}
}
],
"connections": [
{
"fromIndex": 4,
"toIndex": 0,
"type": "NewTempAlarm"
},
{
"fromIndex": 4,
"toIndex": 1,
"type": "ClearTempAlarm"
},
{
"fromIndex": 4,
"toIndex": 2,
"type": "NewHumidityAlarm"
},
{
"fromIndex": 4,
"toIndex": 3,
"type": "ClearHumidityAlarm"
},
{
"fromIndex": 5,
"toIndex": 4,
"type": "Success"
}
],
"ruleChainConnections": null
}
}

32
application/src/main/data/json/system/widget_bundles/gateway_widgets.json
View File

File diff suppressed because one or more lines are too long

25
application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql
View File

@ -123,3 +123,28 @@ BEGIN
END LOOP;
END
$$;
CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint)
LANGUAGE plpgsql AS
$$
DECLARE
ttl_ts bigint;
debug_ttl_ts bigint;
ttl_deleted_count bigint DEFAULT 0;
debug_ttl_deleted_count bigint DEFAULT 0;
BEGIN
IF ttl > 0 THEN
ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint;
EXECUTE format(
'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count;
END IF;
IF debug_ttl > 0 THEN
debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint;
EXECUTE format(
'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count;
END IF;
RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count;
RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count;
deleted := ttl_deleted_count + debug_ttl_deleted_count;
END
$$;

3
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
View File

@ -32,6 +32,7 @@ import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@ -233,6 +234,7 @@ public class ActorSystemContext {
/**
* The following Service will be null if we operate in tb-core mode
*/
@Lazy
@Autowired(required = false)
@Getter
private TbRuleEngineDeviceRpcService tbRuleEngineDeviceRpcService;
@ -240,6 +242,7 @@ public class ActorSystemContext {
/**
* The following Service will be null if we operate in tb-rule-engine mode
*/
@Lazy
@Autowired(required = false)
@Getter
private TbCoreDeviceRpcService tbCoreDeviceRpcService;

3
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
View File

@ -192,6 +192,9 @@ class DefaultTbContext implements TbContext {
@Override
public void ack(TbMsg tbMsg) {
if (nodeCtx.getSelf().isDebugMode()) {
mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), tbMsg, "ACK", null);
}
tbMsg.getCallback().onSuccess();
}

5
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java
View File

@ -249,7 +249,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
if (relationsCount == 0) {
log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId());
if (relationTypes.contains(TbRelationTypes.FAILURE)) {
RuleNodeCtx ruleNodeCtx = nodeActors.get(originatorNodeId);
RuleNodeCtx ruleNodeCtx = nodeActors.get(originatorNodeId);
if (ruleNodeCtx != null) {
msg.getCallback().onFailure(new RuleNodeException(failureMessage, ruleChainName, ruleNodeCtx.getSelf()));
} else {
@ -327,6 +327,9 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
private void pushMsgToNode(RuleNodeCtx nodeCtx, TbMsg msg, String fromRelationType) {
if (nodeCtx != null) {
nodeCtx.getSelfActor().tell(new RuleChainToRuleNodeMsg(new DefaultTbContext(systemContext, nodeCtx), msg, fromRelationType), self);
} else {
log.error("[{}][{}] RuleNodeCtx is empty", entityId, ruleChainName);
msg.getCallback().onFailure(new RuleEngineException("Rule Node CTX is empty"));
}
}

2
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java
View File

@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.common.msg.queue.PartitionChangeMsg;
public class RuleNodeActor extends ComponentActor<RuleNodeId, RuleNodeActorMessageProcessor> {
@ -54,6 +55,7 @@ public class RuleNodeActor extends ComponentActor<RuleNodeId, RuleNodeActorMessa
onStatsPersistTick(id);
break;
case PARTITION_CHANGE_MSG:
onClusterEventMsg((PartitionChangeMsg) msg);
break;
default:
return false;

40
application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java
View File

@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.msg.MsgType;
import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.aware.DeviceAwareMsg;
@ -73,24 +74,30 @@ public class TenantActor extends RuleChainManagerActor {
log.info("[{}] Starting tenant actor.", tenantId);
try {
Tenant tenant = systemContext.getTenantService().findTenantById(tenantId);
// This Service may be started for specific tenant only.
Optional<TenantId> isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant();
if (tenant == null) {
cantFindTenant = true;
log.info("[{}] Started tenant actor for missing tenant.", tenantId);
} else {
// This Service may be started for specific tenant only.
Optional<TenantId> isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant();
isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE);
isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE);
isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE);
isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE);
if (isRuleEngineForCurrentTenant) {
try {
if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) {
initRuleChains();
} else {
isRuleEngineForCurrentTenant = false;
if (isRuleEngineForCurrentTenant) {
try {
if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) {
log.info("[{}] Going to init rule chains", tenantId);
initRuleChains();
} else {
isRuleEngineForCurrentTenant = false;
}
} catch (Exception e) {
cantFindTenant = true;
}
} catch (Exception e) {
cantFindTenant = true;
}
log.info("[{}] Tenant actor started.", tenantId);
}
log.info("[{}] Tenant actor started.", tenantId);
} catch (Exception e) {
log.warn("[{}] Unknown failure", tenantId, e);
}
@ -104,7 +111,12 @@ public class TenantActor extends RuleChainManagerActor {
@Override
protected boolean process(TbActorMsg msg) {
if (cantFindTenant) {
log.info("Missing Tenant msg: {}", msg);
log.info("[{}] Processing missing Tenant msg: {}", tenantId, msg);
if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) {
QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg;
queueMsg.getTbMsg().getCallback().onSuccess();
}
return true;
}
switch (msg.getMsgType()) {
case PARTITION_CHANGE_MSG:

2
application/src/main/java/org/thingsboard/server/controller/QueueController.java
View File

@ -43,7 +43,7 @@ public class QueueController extends BaseController {
ServiceType type = ServiceType.valueOf(serviceType);
switch (type) {
case TB_RULE_ENGINE:
return Arrays.asList("HighPriority", "Main");
return Arrays.asList("Main", "HighPriority", "SequentialByOriginator");
default:
return Collections.emptyList();
}

14
application/src/main/java/org/thingsboard/server/controller/RuleChainController.java
View File

@ -161,15 +161,15 @@ public class RuleChainController extends BaseController {
TenantId tenantId = getCurrentUser().getTenantId();
RuleChain previousRootRuleChain = ruleChainService.getRootTenantRuleChain(tenantId);
if (ruleChainService.setRootRuleChain(getTenantId(), ruleChainId)) {
if (previousRootRuleChain != null) {
previousRootRuleChain = ruleChainService.findRuleChainById(getTenantId(), previousRootRuleChain.getId());
previousRootRuleChain = ruleChainService.findRuleChainById(getTenantId(), previousRootRuleChain.getId());
tbClusterService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(),
ComponentLifecycleEvent.UPDATED);
logEntityAction(previousRootRuleChain.getId(), previousRootRuleChain,
null, ActionType.UPDATED, null);
tbClusterService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(),
ComponentLifecycleEvent.UPDATED);
logEntityAction(previousRootRuleChain.getId(), previousRootRuleChain,
null, ActionType.UPDATED, null);
}
ruleChain = ruleChainService.findRuleChainById(getTenantId(), ruleChainId);
tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(),

11
application/src/main/java/org/thingsboard/server/controller/TelemetryController.java
View File

@ -344,7 +344,7 @@ public class TelemetryController extends BaseController {
return deleteAttributes(entityId, scope, keysStr);
}
private DeferredResult<ResponseEntity> deleteAttributes(EntityId entityIdStr, String scope, String keysStr) throws ThingsboardException {
private DeferredResult<ResponseEntity> deleteAttributes(EntityId entityIdSrc, String scope, String keysStr) throws ThingsboardException {
List<String> keys = toKeysList(keysStr);
if (keys.isEmpty()) {
return getImmediateDeferredResult("Empty keys: " + keysStr, HttpStatus.BAD_REQUEST);
@ -354,13 +354,13 @@ public class TelemetryController extends BaseController {
if (DataConstants.SERVER_SCOPE.equals(scope) ||
DataConstants.SHARED_SCOPE.equals(scope) ||
DataConstants.CLIENT_SCOPE.equals(scope)) {
return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdStr, (result, tenantId, entityId) -> {
return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdSrc, (result, tenantId, entityId) -> {
ListenableFuture<List<Void>> future = attributesService.removeAll(user.getTenantId(), entityId, scope, keys);
Futures.addCallback(future, new FutureCallback<List<Void>>() {
@Override
public void onSuccess(@Nullable List<Void> tmp) {
logAttributesDeleted(user, entityId, scope, keys, null);
if (entityId.getEntityType() == EntityType.DEVICE) {
if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) {
DeviceId deviceId = new DeviceId(entityId.getId());
Set<AttributeKey> keysToNotify = new HashSet<>();
keys.forEach(key -> keysToNotify.add(new AttributeKey(scope, key)));
@ -397,11 +397,6 @@ public class TelemetryController extends BaseController {
@Override
public void onSuccess(@Nullable Void tmp) {
logAttributesUpdated(user, entityId, scope, attributes, null);
if (entityId.getEntityType() == EntityType.DEVICE) {
DeviceId deviceId = new DeviceId(entityId.getId());
tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate(
user.getTenantId(), deviceId, scope, attributes), null);
}
result.setResult(new ResponseEntity(HttpStatus.OK));
}

2
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
View File

@ -23,8 +23,8 @@ import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.service.install.DatabaseTsUpgradeService;
import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService;
import org.thingsboard.server.service.install.DatabaseTsUpgradeService;
import org.thingsboard.server.service.install.EntityDatabaseSchemaService;
import org.thingsboard.server.service.install.SystemDataLoaderService;
import org.thingsboard.server.service.install.TsDatabaseSchemaService;

34
application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java
View File

@ -25,24 +25,32 @@ import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
import org.thingsboard.server.common.data.kv.BooleanDataEntry;
import org.thingsboard.server.common.data.kv.DoubleDataEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.device.DeviceCredentialsService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import java.util.Arrays;
@Service
@Profile("install")
@Slf4j
@ -76,6 +84,9 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
@Autowired
private DeviceService deviceService;
@Autowired
private AttributesService attributesService;
@Autowired
private DeviceCredentialsService deviceCredentialsService;
@ -120,7 +131,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
demoTenant.setRegion("Global");
demoTenant.setTitle("Tenant");
demoTenant = tenantService.saveTenant(demoTenant);
installScripts.createDefaultRuleChains(demoTenant.getId());
installScripts.loadDemoRuleChains(demoTenant.getId());
createUser(Authority.TENANT_ADMIN, demoTenant.getId(), null, "tenant@thingsboard.org", "tenant");
Customer customerA = new Customer();
@ -152,6 +163,25 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
createDevice(demoTenant.getId(), null, DEFAULT_DEVICE_TYPE, "Raspberry Pi Demo Device", "RASPBERRY_PI_DEMO_TOKEN", "Demo device that is used in " +
"Raspberry Pi GPIO control sample application");
DeviceId t1Id = createDevice(demoTenant.getId(), null, "thermostat", "Thermostat T1", "T1_TEST_TOKEN", "Demo device for Thermostats dashboard").getId();
DeviceId t2Id = createDevice(demoTenant.getId(), null, "thermostat", "Thermostat T2", "T2_TEST_TOKEN", "Demo device for Thermostats dashboard").getId();
attributesService.save(demoTenant.getId(), t1Id, DataConstants.SERVER_SCOPE,
Arrays.asList(new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 37.3948)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", -122.1503)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmTemperature", true)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmHumidity", true)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdTemperature", (long) 20)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdHumidity", (long) 50))));
attributesService.save(demoTenant.getId(), t2Id, DataConstants.SERVER_SCOPE,
Arrays.asList(new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 37.493801)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", -121.948769)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmTemperature", true)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmHumidity", true)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdTemperature", (long) 25)),
new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdHumidity", (long) 30))));
installScripts.loadDashboards(demoTenant.getId(), null);
}

27
application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java
View File

@ -24,6 +24,7 @@ import org.springframework.util.StringUtils;
import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
@ -182,4 +183,30 @@ public class InstallScripts {
}
public void loadDemoRuleChains(TenantId tenantId) throws Exception {
Path ruleChainsDir = Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, RULE_CHAINS_DIR);
try {
JsonNode ruleChainJson = objectMapper.readTree(ruleChainsDir.resolve("thermostat_alarms.json").toFile());
RuleChain ruleChain = objectMapper.treeToValue(ruleChainJson.get("ruleChain"), RuleChain.class);
RuleChainMetaData ruleChainMetaData = objectMapper.treeToValue(ruleChainJson.get("metadata"), RuleChainMetaData.class);
ruleChain.setTenantId(tenantId);
ruleChain = ruleChainService.saveRuleChain(ruleChain);
ruleChainMetaData.setRuleChainId(ruleChain.getId());
ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), ruleChainMetaData);
JsonNode rootChainJson = objectMapper.readTree(ruleChainsDir.resolve("root_rule_chain.json").toFile());
RuleChain rootChain = objectMapper.treeToValue(rootChainJson.get("ruleChain"), RuleChain.class);
RuleChainMetaData rootChainMetaData = objectMapper.treeToValue(rootChainJson.get("metadata"), RuleChainMetaData.class);
RuleChainId thermostatsRuleChainId = ruleChain.getId();
rootChainMetaData.getRuleChainConnections().forEach(connection -> connection.setTargetRuleChainId(thermostatsRuleChainId));
rootChain.setTenantId(tenantId);
rootChain = ruleChainService.saveRuleChain(rootChain);
rootChainMetaData.setRuleChainId(rootChain.getId());
ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), rootChainMetaData);
} catch (Exception e) {
log.error("Unable to load dashboard from json", e);
throw new RuntimeException("Unable to load dashboard from json", e);
}
}
}

5
application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java
View File

@ -225,6 +225,11 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
conn.createStatement().execute("ALTER TABLE tenant ADD COLUMN isolated_tb_core boolean DEFAULT (false), ADD COLUMN isolated_tb_rule_engine boolean DEFAULT (false)");
} catch (Exception e) {
}
try {
long ts = System.currentTimeMillis();
conn.createStatement().execute("ALTER TABLE event ADD COLUMN ts bigint DEFAULT " + ts + ";"); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
} catch (Exception e) {
}
log.info("Schema updated.");
}
break;

6
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java
View File

@ -87,6 +87,7 @@ public class DefaultTbClusterService implements TbClusterService {
@Override
public void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback) {
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId());
log.trace("PUSHING msg: {} to:{}", msg, tpi);
byte[] msgBytes = encodingService.encode(msg);
ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToDeviceActorNotificationMsg(ByteString.copyFrom(msgBytes)).build();
producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msg.getDeviceId().getId(), toCoreMsg), callback);
@ -96,6 +97,7 @@ public class DefaultTbClusterService implements TbClusterService {
@Override
public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId);
log.trace("PUSHING msg: {} to:{}", response, tpi);
FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder()
.setRequestIdMSB(response.getId().getMostSignificantBits())
.setRequestIdLSB(response.getId().getLeastSignificantBits())
@ -108,6 +110,7 @@ public class DefaultTbClusterService implements TbClusterService {
@Override
public void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, ToRuleEngineMsg msg, TbQueueCallback callback) {
log.trace("PUSHING msg: {} to:{}", msg, tpi);
producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback);
toRuleEngineMsgs.incrementAndGet();
}
@ -123,6 +126,7 @@ public class DefaultTbClusterService implements TbClusterService {
}
}
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId);
log.trace("PUSHING msg: {} to:{}", tbMsg, tpi);
ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
@ -134,6 +138,7 @@ public class DefaultTbClusterService implements TbClusterService {
@Override
public void pushNotificationToRuleEngine(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId);
log.trace("PUSHING msg: {} to:{}", response, tpi);
FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder()
.setRequestIdMSB(response.getId().getMostSignificantBits())
.setRequestIdLSB(response.getId().getLeastSignificantBits())
@ -147,6 +152,7 @@ public class DefaultTbClusterService implements TbClusterService {
@Override
public void pushNotificationToTransport(String serviceId, ToTransportMsg response, TbQueueCallback callback) {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceId);
log.trace("PUSHING msg: {} to:{}", response, tpi);
producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), response), callback);
toTransportNfs.incrementAndGet();
}

15
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
View File

@ -52,6 +52,7 @@ import org.thingsboard.server.service.subscription.TbSubscriptionUtils;
import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
@ -98,6 +99,11 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
super.init("tb-core-consumer", "tb-core-notifications-consumer");
}
@PreDestroy
public void destroy(){
super.destroy();
}
@Override
public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
if (partitionChangeEvent.getServiceType().equals(getServiceType())) {
@ -117,11 +123,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
}
ConcurrentMap<UUID, TbProtoQueueMsg<ToCoreMsg>> pendingMap = msgs.stream().collect(
Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity()));
ConcurrentMap<UUID, TbProtoQueueMsg<ToCoreMsg>> failedMap = new ConcurrentHashMap<>();
CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
TbPackProcessingContext<TbProtoQueueMsg<ToCoreMsg>> ctx = new TbPackProcessingContext<>(
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
pendingMap.forEach((id, msg) -> {
log.trace("[{}] Creating main callback for message: {}", id, msg.getValue());
TbCallback callback = new TbPackCallback<>(id, processingTimeoutLatch, pendingMap, failedMap);
TbCallback callback = new TbPackCallback<>(id, ctx);
try {
ToCoreMsg toCoreMsg = msg.getValue();
if (toCoreMsg.hasToSubscriptionMgrMsg()) {
@ -147,8 +154,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
}
});
if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) {
pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue()));
failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue()));
ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue()));
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue()));
}
mainConsumer.commit();
} catch (Exception e) {

10
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
View File

@ -31,7 +31,6 @@ import org.thingsboard.server.common.msg.queue.ServiceQueue;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.queue.TbQueueConsumer;
@ -64,9 +63,6 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.stream.Collectors;
@Service
@TbRuleEngineComponent
@ -116,10 +112,10 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
@PreDestroy
public void stop() {
super.destroy();
if (submitExecutor != null) {
submitExecutor.shutdownNow();
}
ruleEngineSettings.getQueues().forEach(config -> consumerConfigurations.put(config.getName(), config));
}
@ -156,7 +152,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
submitStrategy.init(msgs);
while (!stopped) {
ProcessingAttemptContext ctx = new ProcessingAttemptContext(submitStrategy);
TbMsgPackProcessingContext ctx = new TbMsgPackProcessingContext(submitStrategy);
submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> {
log.trace("[{}] Creating callback for message: {}", id, msg.getValue());
ToRuleEngineMsg toRuleEngineMsg = msg.getValue();
@ -174,7 +170,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
}));
boolean timeout = false;
if (!ctx.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) {
if (!ctx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS)) {
timeout = true;
}

2
application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java
View File

@ -79,7 +79,7 @@ public class TbCoreConsumerStats {
public void printStats() {
int total = totalCounter.getAndSet(0);
if (total > 0) {
log.info("Transport total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]" +
log.info("Total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]" +
" deviceState [{}] subMgr [{}] coreNfs [{}]",
total, sessionEventCounter.getAndSet(0),
getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0),

8
application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java
View File

@ -18,21 +18,17 @@ package org.thingsboard.server.service.queue;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.common.msg.queue.RuleNodeException;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import java.util.UUID;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
@Slf4j
public class TbMsgPackCallback implements TbMsgCallback {
private final UUID id;
private final TenantId tenantId;
private final ProcessingAttemptContext ctx;
private final TbMsgPackProcessingContext ctx;
public TbMsgPackCallback(UUID id, TenantId tenantId, ProcessingAttemptContext ctx) {
public TbMsgPackCallback(UUID id, TenantId tenantId, TbMsgPackProcessingContext ctx) {
this.id = id;
this.tenantId = tenantId;
this.ctx = ctx;

25
application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java → application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java
View File

@ -27,11 +27,13 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
public class ProcessingAttemptContext {
public class TbMsgPackProcessingContext {
private final TbRuleEngineSubmitStrategy submitStrategy;
private final AtomicInteger pendingCount;
private final CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
@Getter
private final ConcurrentMap<UUID, TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> pendingMap;
@ -42,9 +44,10 @@ public class ProcessingAttemptContext {
@Getter
private final ConcurrentMap<TenantId, RuleEngineException> exceptionsMap = new ConcurrentHashMap<>();
public ProcessingAttemptContext(TbRuleEngineSubmitStrategy submitStrategy) {
public TbMsgPackProcessingContext(TbRuleEngineSubmitStrategy submitStrategy) {
this.submitStrategy = submitStrategy;
this.pendingMap = submitStrategy.getPendingMap();
this.pendingCount = new AtomicInteger(pendingMap.size());
}
public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException {
@ -54,16 +57,12 @@ public class ProcessingAttemptContext {
public void onSuccess(UUID id) {
TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg> msg;
boolean empty = false;
synchronized (pendingMap) {
msg = pendingMap.remove(id);
if (msg != null) {
empty = pendingMap.isEmpty();
}
}
msg = pendingMap.remove(id);
if (msg != null) {
empty = pendingCount.decrementAndGet() == 0;
successMap.put(id, msg);
submitStrategy.onSuccess(id);
}
submitStrategy.onSuccess(id);
if (empty) {
processingTimeoutLatch.countDown();
}
@ -72,13 +71,9 @@ public class ProcessingAttemptContext {
public void onFailure(TenantId tenantId, UUID id, RuleEngineException e) {
TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg> msg;
boolean empty = false;
synchronized (pendingMap) {
msg = pendingMap.remove(id);
if (msg != null) {
empty = pendingMap.isEmpty();
}
}
msg = pendingMap.remove(id);
if (msg != null) {
empty = pendingCount.decrementAndGet() == 0;
failedMap.put(id, msg);
exceptionsMap.putIfAbsent(tenantId, e);
}

28
application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java
View File

@ -19,44 +19,26 @@ import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.msg.queue.TbCallback;
import java.util.UUID;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
@Slf4j
public class TbPackCallback<T> implements TbCallback {
private final CountDownLatch processingTimeoutLatch;
private final ConcurrentMap<UUID, T> ackMap;
private final ConcurrentMap<UUID, T> failedMap;
private final TbPackProcessingContext<T> ctx;
private final UUID id;
public TbPackCallback(UUID id,
CountDownLatch processingTimeoutLatch,
ConcurrentMap<UUID, T> ackMap,
ConcurrentMap<UUID, T> failedMap) {
public TbPackCallback(UUID id, TbPackProcessingContext<T> ctx) {
this.id = id;
this.processingTimeoutLatch = processingTimeoutLatch;
this.ackMap = ackMap;
this.failedMap = failedMap;
this.ctx = ctx;
}
@Override
public void onSuccess() {
log.trace("[{}] ON SUCCESS", id);
T msg = ackMap.remove(id);
if (msg != null && ackMap.isEmpty()) {
processingTimeoutLatch.countDown();
}
ctx.onSuccess(id);
}
@Override
public void onFailure(Throwable t) {
log.trace("[{}] ON FAILURE", id, t);
T msg = ackMap.remove(id);
if (msg != null) {
failedMap.put(id, msg);
}
if (ackMap.isEmpty()) {
processingTimeoutLatch.countDown();
}
ctx.onFailure(id, t);
}
}

90
application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java
View File

@ -0,0 +1,90 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.queue;
import lombok.extern.slf4j.Slf4j;
import java.util.UUID;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@Slf4j
public class TbPackProcessingContext<T> {
private final AtomicInteger pendingCount;
private final CountDownLatch processingTimeoutLatch;
private final ConcurrentMap<UUID, T> ackMap;
private final ConcurrentMap<UUID, T> failedMap;
public TbPackProcessingContext(CountDownLatch processingTimeoutLatch,
ConcurrentMap<UUID, T> ackMap,
ConcurrentMap<UUID, T> failedMap) {
this.processingTimeoutLatch = processingTimeoutLatch;
this.pendingCount = new AtomicInteger(ackMap.size());
this.ackMap = ackMap;
this.failedMap = failedMap;
}
public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException {
return processingTimeoutLatch.await(packProcessingTimeout, milliseconds);
}
public void onSuccess(UUID id) {
boolean empty = false;
T msg = ackMap.remove(id);
if (msg != null) {
empty = pendingCount.decrementAndGet() == 0;
}
if (empty) {
processingTimeoutLatch.countDown();
} else {
if (log.isTraceEnabled()) {
log.trace("Items left: {}", ackMap.size());
for (T t : ackMap.values()) {
log.trace("left item: {}", t);
}
}
}
}
public void onFailure(UUID id, Throwable t) {
boolean empty = false;
T msg = ackMap.remove(id);
if (msg != null) {
empty = pendingCount.decrementAndGet() == 0;
failedMap.put(id, msg);
if (log.isTraceEnabled()) {
log.trace("Items left: {}", ackMap.size());
for (T v : ackMap.values()) {
log.trace("left item: {}", v);
}
}
}
if (empty) {
processingTimeoutLatch.countDown();
}
}
public ConcurrentMap<UUID, T> getAckMap() {
return ackMap;
}
public ConcurrentMap<UUID, T> getFailedMap() {
return failedMap;
}
}

11
application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java
View File

@ -23,11 +23,13 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.service.encoding.DataDecodingEncodingService;
import org.thingsboard.server.service.queue.TbPackCallback;
import org.thingsboard.server.service.queue.TbPackProcessingContext;
import javax.annotation.PreDestroy;
import java.util.List;
@ -92,11 +94,12 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
}
ConcurrentMap<UUID, TbProtoQueueMsg<N>> pendingMap = msgs.stream().collect(
Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity()));
ConcurrentMap<UUID, TbProtoQueueMsg<N>> failedMap = new ConcurrentHashMap<>();
CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
TbPackProcessingContext<TbProtoQueueMsg<N>> ctx = new TbPackProcessingContext<>(
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
pendingMap.forEach((id, msg) -> {
log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue());
TbCallback callback = new TbPackCallback<>(id, processingTimeoutLatch, pendingMap, failedMap);
TbCallback callback = new TbPackCallback<>(id, ctx);
try {
handleNotification(id, msg, callback);
} catch (Throwable e) {
@ -105,8 +108,8 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
}
});
if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) {
pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue()));
failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue()));
ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue()));
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue()));
}
nfConsumer.commit();
} catch (Exception e) {

6
application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java
View File

@ -20,7 +20,7 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.service.queue.ProcessingAttemptContext;
import org.thingsboard.server.service.queue.TbMsgPackProcessingContext;
import java.util.UUID;
import java.util.concurrent.ConcurrentMap;
@ -32,9 +32,9 @@ public class TbRuleEngineProcessingResult {
@Getter
private final boolean timeout;
@Getter
private final ProcessingAttemptContext ctx;
private final TbMsgPackProcessingContext ctx;
public TbRuleEngineProcessingResult(boolean timeout, ProcessingAttemptContext ctx) {
public TbRuleEngineProcessingResult(boolean timeout, TbMsgPackProcessingContext ctx) {
this.timeout = timeout;
this.ctx = ctx;
this.success = !timeout && ctx.getPendingMap().isEmpty() && ctx.getFailedMap().isEmpty();

16
application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java
View File

@ -17,6 +17,8 @@ package org.thingsboard.server.service.queue.processing;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueAckStrategyConfiguration;
@ -32,7 +34,7 @@ public class TbRuleEngineProcessingStrategyFactory {
public TbRuleEngineProcessingStrategy newInstance(String name, TbRuleEngineQueueAckStrategyConfiguration configuration) {
switch (configuration.getType()) {
case "SKIP_ALL":
case "SKIP_ALL_FAILURES":
return new SkipStrategy(name);
case "RETRY_ALL":
return new RetryStrategy(name, true, true, true, configuration);
@ -98,7 +100,7 @@ public class TbRuleEngineProcessingStrategyFactory {
}
log.info("[{}] Going to reprocess {} messages", queueName, toReprocess.size());
if (log.isTraceEnabled()) {
toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, msg.getValue()));
toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
}
if (pauseBetweenRetries > 0) {
try {
@ -123,7 +125,15 @@ public class TbRuleEngineProcessingStrategyFactory {
@Override
public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) {
log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size());
if (!result.isSuccess()) {
log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size());
}
if (log.isTraceEnabled()) {
result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
}
if (log.isTraceEnabled()) {
result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
}
return new TbRuleEngineProcessingDecision(true, null);
}
}

4
application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java
View File

@ -29,9 +29,9 @@ public class TbRuleEngineSubmitStrategyFactory {
return new BurstTbRuleEngineSubmitStrategy(name);
case "BATCH":
return new BatchTbRuleEngineSubmitStrategy(name, configuration.getBatchSize());
case "SEQUENTIAL_WITHIN_ORIGINATOR":
case "SEQUENTIAL_BY_ORIGINATOR":
return new SequentialByOriginatorIdTbRuleEngineSubmitStrategy(name);
case "SEQUENTIAL_WITHIN_TENANT":
case "SEQUENTIAL_BY_TENANT":
return new SequentialByTenantIdTbRuleEngineSubmitStrategy(name);
case "SEQUENTIAL":
return new SequentialTbRuleEngineSubmitStrategy(name);

10
application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java
View File

@ -106,7 +106,7 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService {
@Override
public void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response) {
log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId());
log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), response);
UUID requestId = response.getId();
Consumer<FromDeviceRpcResponse> consumer = localToRuleEngineRpcRequests.remove(requestId);
if (consumer != null) {
@ -177,9 +177,9 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService {
private void scheduleToRuleEngineTimeout(ToDeviceRpcRequest request, UUID requestId) {
long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis());
log.trace("[{}] processing to rule engine request: [{}]", this.hashCode(), requestId);
log.trace("[{}] processing to rule engine request.", requestId);
scheduler.schedule(() -> {
log.trace("[{}] timeout for to rule engine request: [{}]", this.hashCode(), requestId);
log.trace("[{}] timeout for processing to rule engine request.", requestId);
Consumer<FromDeviceRpcResponse> consumer = localToRuleEngineRpcRequests.remove(requestId);
if (consumer != null) {
consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT));
@ -189,9 +189,9 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService {
private void scheduleToDeviceTimeout(ToDeviceRpcRequest request, UUID requestId) {
long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis());
log.trace("[{}] processing to device request: [{}]", this.hashCode(), requestId);
log.trace("[{}] processing to device request.", requestId);
scheduler.schedule(() -> {
log.trace("[{}] timeout for to device request: [{}]", this.hashCode(), requestId);
log.trace("[{}] timeout for to device request.", requestId);
localToDeviceRpcRequests.remove(requestId);
}, timeout, TimeUnit.MILLISECONDS);
}

8
application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java
View File

@ -129,10 +129,10 @@ public class DefaultDeviceStateService implements DeviceStateService {
private volatile boolean clusterUpdatePending = false;
private ListeningScheduledExecutorService queueExecutor;
private ConcurrentMap<TopicPartitionInfo, Set<DeviceId>> partitionedDevices = new ConcurrentHashMap<>();
private ConcurrentMap<DeviceId, DeviceStateData> deviceStates = new ConcurrentHashMap<>();
private ConcurrentMap<DeviceId, Long> deviceLastReportedActivity = new ConcurrentHashMap<>();
private ConcurrentMap<DeviceId, Long> deviceLastSavedActivity = new ConcurrentHashMap<>();
private final ConcurrentMap<TopicPartitionInfo, Set<DeviceId>> partitionedDevices = new ConcurrentHashMap<>();
private final ConcurrentMap<DeviceId, DeviceStateData> deviceStates = new ConcurrentHashMap<>();
private final ConcurrentMap<DeviceId, Long> deviceLastReportedActivity = new ConcurrentHashMap<>();
private final ConcurrentMap<DeviceId, Long> deviceLastSavedActivity = new ConcurrentHashMap<>();
public DefaultDeviceStateService(TenantService tenantService, DeviceService deviceService,
AttributesService attributesService, TimeseriesService tsService,

2
application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java
View File

@ -41,6 +41,6 @@ public interface DeviceStateService extends ApplicationListener<PartitionChangeE
void onDeviceInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout);
void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto serverAddress, TbCallback bytes);
void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto proto, TbCallback bytes);
}

11
application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java
View File

@ -47,6 +47,7 @@ import java.util.stream.Collectors;
@Slf4j
public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService {
public static final String TB_SERVICE_QUEUE = "TbServiceQueue";
public static final FutureCallback<Void> CALLBACK = new FutureCallback<Void>() {
@Override
public void onSuccess(@Nullable Void result) {
@ -95,7 +96,13 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS
});
ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> {
TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString()));
tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK);
try {
tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK);
} catch (DataValidationException e2) {
if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) {
throw e2;
}
}
});
ruleEngineStats.reset();
}
@ -113,7 +120,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS
asset = new Asset();
asset.setTenantId(tenantId);
asset.setName(queueName + "_" + serviceInfoProvider.getServiceId());
asset.setType("TbServiceQueue");
asset.setType(TB_SERVICE_QUEUE);
asset = assetService.saveAsset(asset);
}
assetId = asset.getId();

8
application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java
View File

@ -345,6 +345,8 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
keys.forEach(key -> subState.put(key, 0L));
attributesData.forEach(v -> subState.put(v.getKey(), v.getTs()));
TbAttributeSubscriptionScope scope = StringUtils.isEmpty(cmd.getScope()) ? TbAttributeSubscriptionScope.SERVER_SCOPE : TbAttributeSubscriptionScope.valueOf(cmd.getScope());
TbAttributeSubscription sub = TbAttributeSubscription.builder()
.serviceId(serviceId)
.sessionId(sessionId)
@ -353,7 +355,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
.entityId(entityId)
.allKeys(false)
.keyStates(subState)
.scope(TbAttributeSubscriptionScope.valueOf(cmd.getScope())).build();
.scope(scope).build();
subService.addSubscription(sub);
}
@ -440,6 +442,8 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
Map<String, Long> subState = new HashMap<>(attributesData.size());
attributesData.forEach(v -> subState.put(v.getKey(), v.getTs()));
TbAttributeSubscriptionScope scope = StringUtils.isEmpty(cmd.getScope()) ? TbAttributeSubscriptionScope.SERVER_SCOPE : TbAttributeSubscriptionScope.valueOf(cmd.getScope());
TbAttributeSubscription sub = TbAttributeSubscription.builder()
.serviceId(serviceId)
.sessionId(sessionId)
@ -448,7 +452,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
.entityId(entityId)
.allKeys(true)
.keyStates(subState)
.scope(TbAttributeSubscriptionScope.valueOf(cmd.getScope())).build();
.scope(scope).build();
subService.addSubscription(sub);
}

40
application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java → application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java
View File

@ -17,47 +17,27 @@ package org.thingsboard.server.service.ttl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.thingsboard.server.dao.util.PsqlTsAnyDao;
import org.thingsboard.server.dao.util.PsqlDao;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.SQLWarning;
import java.sql.Statement;
@PsqlTsAnyDao
@Slf4j
public abstract class AbstractTimeseriesCleanUpService {
@Value("${sql.ttl.ts_key_value_ttl}")
protected long systemTtl;
@Value("${sql.ttl.enabled}")
private boolean ttlTaskExecutionEnabled;
@Slf4j
@PsqlDao
public abstract class AbstractCleanUpService {
@Value("${spring.datasource.url}")
private String dbUrl;
protected String dbUrl;
@Value("${spring.datasource.username}")
private String dbUserName;
protected String dbUserName;
@Value("${spring.datasource.password}")
private String dbPassword;
@Scheduled(initialDelayString = "${sql.ttl.execution_interval_ms}", fixedDelayString = "${sql.ttl.execution_interval_ms}")
public void cleanUp() {
if (ttlTaskExecutionEnabled) {
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
doCleanUp(conn);
} catch (SQLException e) {
log.error("SQLException occurred during TTL task execution ", e);
}
}
}
protected abstract void doCleanUp(Connection connection);
protected String dbPassword;
protected long executeQuery(Connection conn, String query) {
long removed = 0L;
@ -74,7 +54,7 @@ public abstract class AbstractTimeseriesCleanUpService {
return removed;
}
private void getWarnings(Statement statement) throws SQLException {
protected void getWarnings(Statement statement) throws SQLException {
SQLWarning warnings = statement.getWarnings();
if (warnings != null) {
log.debug("{}", warnings.getMessage());
@ -86,4 +66,6 @@ public abstract class AbstractTimeseriesCleanUpService {
}
}
}
protected abstract void doCleanUp(Connection connection);
}

59
application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java
View File

@ -0,0 +1,59 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.ttl.events;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.PsqlDao;
import org.thingsboard.server.service.ttl.AbstractCleanUpService;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
@PsqlDao
@Slf4j
@Service
public class EventsCleanUpService extends AbstractCleanUpService {
@Value("${sql.ttl.events.events_ttl}")
private long ttl;
@Value("${sql.ttl.events.debug_events_ttl}")
private long debugTtl;
@Value("${sql.ttl.events.enabled}")
private boolean ttlTaskExecutionEnabled;
@Scheduled(initialDelayString = "${sql.ttl.events.execution_interval_ms}", fixedDelayString = "${sql.ttl.events.execution_interval_ms}")
public void cleanUp() {
if (ttlTaskExecutionEnabled) {
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
doCleanUp(conn);
} catch (SQLException e) {
log.error("SQLException occurred during TTL task execution ", e);
}
}
}
@Override
protected void doCleanUp(Connection connection) {
long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", " + debugTtl + ", 0);");
log.info("Total events removed by TTL: [{}]", totalEventsRemoved);
}
}

49
application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java
View File

@ -0,0 +1,49 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.ttl.timeseries;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.thingsboard.server.dao.util.PsqlTsAnyDao;
import org.thingsboard.server.service.ttl.AbstractCleanUpService;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
@PsqlTsAnyDao
@Slf4j
public abstract class AbstractTimeseriesCleanUpService extends AbstractCleanUpService {
@Value("${sql.ttl.ts.ts_key_value_ttl}")
protected long systemTtl;
@Value("${sql.ttl.ts.enabled}")
private boolean ttlTaskExecutionEnabled;
@Scheduled(initialDelayString = "${sql.ttl.ts.execution_interval_ms}", fixedDelayString = "${sql.ttl.ts.execution_interval_ms}")
public void cleanUp() {
if (ttlTaskExecutionEnabled) {
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
doCleanUp(conn);
} catch (SQLException e) {
log.error("SQLException occurred during TTL task execution ", e);
}
}
}
}

2
application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java → application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java
View File

@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.ttl;
package org.thingsboard.server.service.ttl.timeseries;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;

2
application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java → application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java
View File

@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.ttl;
package org.thingsboard.server.service.ttl.timeseries;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;

3
application/src/main/resources/logback.xml
View File

@ -31,8 +31,11 @@
<!-- <logger name="org.thingsboard.server.service.queue" level="TRACE" />-->
<!-- <logger name="org.thingsboard.server.service.transport" level="TRACE" />-->
<logger name="com.microsoft.azure.servicebus.primitives.CoreMessageReceiver" level="OFF" />
<root level="INFO">
<appender-ref ref="STDOUT"/>
</root>
</configuration>

107
application/src/main/resources/thingsboard.yml
View File

@ -181,31 +181,37 @@ cassandra:
# SQL configuration parameters
sql:
# Specify batch size for persisting attribute updates
attributes:
batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}"
batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}"
stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}"
ts:
batch_size: "${SQL_TS_BATCH_SIZE:10000}"
batch_max_delay: "${SQL_TS_BATCH_MAX_DELAY_MS:100}"
stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}"
ts_latest:
batch_size: "${SQL_TS_LATEST_BATCH_SIZE:10000}"
batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:100}"
stats_print_interval_ms: "${SQL_TS_LATEST_BATCH_STATS_PRINT_MS:10000}"
# Specify whether to remove null characters from strValue of attributes and timeseries before insert
remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}"
postgres:
# Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE.
ts_key_value_partitioning: "${SQL_POSTGRES_TS_KV_PARTITIONING:MONTHS}"
timescale:
# Specify Interval size for new data chunks storage.
chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}"
ttl:
enabled: "${SQL_TTL_ENABLED:true}"
execution_interval_ms: "${SQL_TTL_EXECUTION_INTERVAL:86400000}" # Number of miliseconds
ts_key_value_ttl: "${SQL_TTL_TS_KEY_VALUE_TTL:0}" # Number of seconds
# Specify batch size for persisting attribute updates
attributes:
batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}"
batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}"
stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}"
ts:
batch_size: "${SQL_TS_BATCH_SIZE:10000}"
batch_max_delay: "${SQL_TS_BATCH_MAX_DELAY_MS:100}"
stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}"
ts_latest:
batch_size: "${SQL_TS_LATEST_BATCH_SIZE:10000}"
batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:100}"
stats_print_interval_ms: "${SQL_TS_LATEST_BATCH_STATS_PRINT_MS:10000}"
# Specify whether to remove null characters from strValue of attributes and timeseries before insert
remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}"
postgres:
# Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE.
ts_key_value_partitioning: "${SQL_POSTGRES_TS_KV_PARTITIONING:MONTHS}"
timescale:
# Specify Interval size for new data chunks storage.
chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}"
ttl:
ts:
enabled: "${SQL_TTL_TS_ENABLED:true}"
execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day
ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds
events:
enabled: "${SQL_TTL_EVENTS_ENABLED:true}"
execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day
events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" # Number of seconds
debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" # Number of seconds. The current value corresponds to one week
# Actor system parameters
actors:
@ -410,12 +416,13 @@ audit-log:
password: "${AUDIT_LOG_SINK_PASSWORD:}"
state:
defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:10}"
defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:10}"
# Should be greater then transport.sessions.report_timeout
defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:600}"
defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:60}"
persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}"
js:
evaluator: "${JS_EVALUATOR:remote}" # local/remote
evaluator: "${JS_EVALUATOR:local}" # local/remote
# Built-in JVM JavaScript environment properties
local:
# Use Sandboxed (secured) JVM JavaScript environment
@ -543,14 +550,25 @@ queue:
pubsub:
project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}"
service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}"
ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again
max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes
max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}"
queue-properties:
rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}"
core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}"
transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}"
notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}"
js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}"
service_bus:
namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}"
sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}"
sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}"
max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}"
queue-properties:
rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}"
core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}"
transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}"
notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}"
js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}"
rabbitmq:
exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}"
host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}"
@ -561,6 +579,12 @@ queue:
automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}"
connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}"
handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}"
queue-properties:
rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}"
core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}"
transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}"
notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}"
js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}"
partitions:
hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}"
virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}"
@ -578,7 +602,7 @@ queue:
partitions: "${TB_QUEUE_CORE_PARTITIONS:10}"
pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}"
stats:
enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}"
enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}"
print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}"
js:
# JS Eval request topic
@ -607,11 +631,11 @@ queue:
partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}"
pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}"
submit-strategy:
type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL
type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL
# For BATCH only
batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch
processing-strategy:
type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT
type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:SKIP_ALL_FAILURES}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT
# For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT
retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited
failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages;
@ -619,10 +643,10 @@ queue:
- name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}"
topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}"
poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}"
partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}"
partitions: "${TB_QUEUE_RE_HP_PARTITIONS:10}"
pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}"
submit-strategy:
type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL
type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL
# For BATCH only
batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch
processing-strategy:
@ -631,6 +655,21 @@ queue:
retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited
failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages;
pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries;
- name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}"
topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}"
poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}"
partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:10}"
pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:60000}"
submit-strategy:
type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL
# For BATCH only
batch-size: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch
processing-strategy:
type: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT
# For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT
retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited
failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages;
pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries;
transport:
# For high priority notifications that require minimum latency and processing time
notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}"

128
application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java
View File

@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.page.TextPageData;
import org.thingsboard.server.common.data.page.TextPageLink;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.service.stats.DefaultRuleEngineStatisticsService;
import java.util.ArrayList;
import java.util.Collections;
@ -71,7 +72,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
public void afterTest() throws Exception {
loginSysAdmin();
doDelete("/api/tenant/"+savedTenant.getId().getId().toString())
doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
.andExpect(status().isOk());
}
@ -111,26 +112,27 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
@Test
public void testFindAssetTypesByTenantId() throws Exception {
List<Asset> assets = new ArrayList<>();
for (int i=0;i<3;i++) {
for (int i = 0; i < 3; i++) {
Asset asset = new Asset();
asset.setName("My asset B"+i);
asset.setName("My asset B" + i);
asset.setType("typeB");
assets.add(doPost("/api/asset", asset, Asset.class));
}
for (int i=0;i<7;i++) {
for (int i = 0; i < 7; i++) {
Asset asset = new Asset();
asset.setName("My asset C"+i);
asset.setName("My asset C" + i);
asset.setType("typeC");
assets.add(doPost("/api/asset", asset, Asset.class));
}
for (int i=0;i<9;i++) {
for (int i = 0; i < 9; i++) {
Asset asset = new Asset();
asset.setName("My asset A"+i);
asset.setName("My asset A" + i);
asset.setType("typeA");
assets.add(doPost("/api/asset", asset, Asset.class));
}
List<EntitySubtype> assetTypes = doGetTyped("/api/asset/types",
new TypeReference<List<EntitySubtype>>(){});
new TypeReference<List<EntitySubtype>>() {
});
Assert.assertNotNull(assetTypes);
Assert.assertEquals(3, assetTypes.size());
@ -146,10 +148,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
asset.setType("default");
Asset savedAsset = doPost("/api/asset", asset, Asset.class);
doDelete("/api/asset/"+savedAsset.getId().getId().toString())
doDelete("/api/asset/" + savedAsset.getId().getId().toString())
.andExpect(status().isOk());
doGet("/api/asset/"+savedAsset.getId().getId().toString())
doGet("/api/asset/" + savedAsset.getId().getId().toString())
.andExpect(status().isNotFound());
}
@ -244,16 +246,16 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
loginSysAdmin();
doDelete("/api/tenant/"+savedTenant2.getId().getId().toString())
doDelete("/api/tenant/" + savedTenant2.getId().getId().toString())
.andExpect(status().isOk());
}
@Test
public void testFindTenantAssets() throws Exception {
List<Asset> assets = new ArrayList<>();
for (int i=0;i<178;i++) {
for (int i = 0; i < 178; i++) {
Asset asset = new Asset();
asset.setName("Asset"+i);
asset.setName("Asset" + i);
asset.setType("default");
assets.add(doPost("/api/asset", asset, Asset.class));
}
@ -262,13 +264,16 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
TextPageData<Asset> pageData = null;
do {
pageData = doGetTypedWithPageLink("/api/tenant/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink);
new TypeReference<TextPageData<Asset>>() {
}, pageLink);
loadedAssets.addAll(pageData.getData());
if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink();
}
} while (pageData.hasNext());
loadedAssets.removeIf(asset -> asset.getType().equals(DefaultRuleEngineStatisticsService.TB_SERVICE_QUEUE));
Collections.sort(assets, idComparator);
Collections.sort(loadedAssets, idComparator);
@ -279,10 +284,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
public void testFindTenantAssetsByName() throws Exception {
String title1 = "Asset title 1";
List<Asset> assetsTitle1 = new ArrayList<>();
for (int i=0;i<143;i++) {
for (int i = 0; i < 143; i++) {
Asset asset = new Asset();
String suffix = RandomStringUtils.randomAlphanumeric(15);
String name = title1+suffix;
String name = title1 + suffix;
name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase();
asset.setName(name);
asset.setType("default");
@ -290,10 +295,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
}
String title2 = "Asset title 2";
List<Asset> assetsTitle2 = new ArrayList<>();
for (int i=0;i<75;i++) {
for (int i = 0; i < 75; i++) {
Asset asset = new Asset();
String suffix = RandomStringUtils.randomAlphanumeric(15);
String name = title2+suffix;
String name = title2 + suffix;
name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase();
asset.setName(name);
asset.setType("default");
@ -305,7 +310,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
TextPageData<Asset> pageData = null;
do {
pageData = doGetTypedWithPageLink("/api/tenant/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink);
new TypeReference<TextPageData<Asset>>() {
}, pageLink);
loadedAssetsTitle1.addAll(pageData.getData());
if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink();
@ -321,7 +327,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
pageLink = new TextPageLink(4, title2);
do {
pageData = doGetTypedWithPageLink("/api/tenant/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink);
new TypeReference<TextPageData<Asset>>() {
}, pageLink);
loadedAssetsTitle2.addAll(pageData.getData());
if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink();
@ -334,24 +341,26 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
Assert.assertEquals(assetsTitle2, loadedAssetsTitle2);
for (Asset asset : loadedAssetsTitle1) {
doDelete("/api/asset/"+asset.getId().getId().toString())
doDelete("/api/asset/" + asset.getId().getId().toString())
.andExpect(status().isOk());
}
pageLink = new TextPageLink(4, title1);
pageData = doGetTypedWithPageLink("/api/tenant/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink);
new TypeReference<TextPageData<Asset>>() {
}, pageLink);
Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size());
for (Asset asset : loadedAssetsTitle2) {
doDelete("/api/asset/"+asset.getId().getId().toString())
doDelete("/api/asset/" + asset.getId().getId().toString())
.andExpect(status().isOk());
}
pageLink = new TextPageLink(4, title2);
pageData = doGetTypedWithPageLink("/api/tenant/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink);
new TypeReference<TextPageData<Asset>>() {
}, pageLink);
Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size());
}
@ -361,10 +370,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
String title1 = "Asset title 1";
String type1 = "typeA";
List<Asset> assetsType1 = new ArrayList<>();
for (int i=0;i<143;i++) {
for (int i = 0; i < 143; i++) {
Asset asset = new Asset();
String suffix = RandomStringUtils.randomAlphanumeric(15);
String name = title1+suffix;
String name = title1 + suffix;
name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase();
asset.setName(name);
asset.setType(type1);
@ -373,10 +382,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
String title2 = "Asset title 2";
String type2 = "typeB";
List<Asset> assetsType2 = new ArrayList<>();
for (int i=0;i<75;i++) {
for (int i = 0; i < 75; i++) {
Asset asset = new Asset();
String suffix = RandomStringUtils.randomAlphanumeric(15);
String name = title2+suffix;
String name = title2 + suffix;
name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase();
asset.setName(name);
asset.setType(type2);
@ -388,7 +397,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
TextPageData<Asset> pageData = null;
do {
pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&",
new TypeReference<TextPageData<Asset>>(){}, pageLink, type1);
new TypeReference<TextPageData<Asset>>() {
}, pageLink, type1);
loadedAssetsType1.addAll(pageData.getData());
if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink();
@ -404,7 +414,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
pageLink = new TextPageLink(4);
do {
pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&",
new TypeReference<TextPageData<Asset>>(){}, pageLink, type2);
new TypeReference<TextPageData<Asset>>() {
}, pageLink, type2);
loadedAssetsType2.addAll(pageData.getData());
if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink();
@ -417,24 +428,26 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
Assert.assertEquals(assetsType2, loadedAssetsType2);
for (Asset asset : loadedAssetsType1) {
doDelete("/api/asset/"+asset.getId().getId().toString())
doDelete("/api/asset/" + asset.getId().getId().toString())
.andExpect(status().isOk());
}
pageLink = new TextPageLink(4);
pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&",
new TypeReference<TextPageData<Asset>>(){}, pageLink, type1);
new TypeReference<TextPageData<Asset>>() {
}, pageLink, type1);
Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size());
for (Asset asset : loadedAssetsType2) {
doDelete("/api/asset/"+asset.getId().getId().toString())
doDelete("/api/asset/" + asset.getId().getId().toString())
.andExpect(status().isOk());
}
pageLink = new TextPageLink(4);
pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&",
new TypeReference<TextPageData<Asset>>(){}, pageLink, type2);
new TypeReference<TextPageData<Asset>>() {
}, pageLink, type2);
Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size());
}
@ -447,9 +460,9 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
CustomerId customerId = customer.getId();
List<Asset> assets = new ArrayList<>();
for (int i=0;i<128;i++) {
for (int i = 0; i < 128; i++) {
Asset asset = new Asset();
asset.setName("Asset"+i);
asset.setName("Asset" + i);
asset.setType("default");
asset = doPost("/api/asset", asset, Asset.class);
assets.add(doPost("/api/customer/" + customerId.getId().toString()
@ -461,7 +474,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
TextPageData<Asset> pageData = null;
do {
pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink);
new TypeReference<TextPageData<Asset>>() {
}, pageLink);
loadedAssets.addAll(pageData.getData());
if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink();
@ -483,10 +497,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
String title1 = "Asset title 1";
List<Asset> assetsTitle1 = new ArrayList<>();
for (int i=0;i<125;i++) {
for (int i = 0; i < 125; i++) {
Asset asset = new Asset();
String suffix = RandomStringUtils.randomAlphanumeric(15);
String name = title1+suffix;
String name = title1 + suffix;
name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase();
asset.setName(name);
asset.setType("default");
@ -496,10 +510,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
}
String title2 = "Asset title 2";
List<Asset> assetsTitle2 = new ArrayList<>();
for (int i=0;i<143;i++) {
for (int i = 0; i < 143; i++) {
Asset asset = new Asset();
String suffix = RandomStringUtils.randomAlphanumeric(15);
String name = title2+suffix;
String name = title2 + suffix;
name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase();
asset.setName(name);
asset.setType("default");
@ -513,7 +527,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
TextPageData<Asset> pageData = null;
do {
pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink);
new TypeReference<TextPageData<Asset>>() {
}, pageLink);
loadedAssetsTitle1.addAll(pageData.getData());
if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink();
@ -529,7 +544,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
pageLink = new TextPageLink(4, title2);
do {
pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink);
new TypeReference<TextPageData<Asset>>() {
}, pageLink);
loadedAssetsTitle2.addAll(pageData.getData());
if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink();
@ -548,7 +564,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
pageLink = new TextPageLink(4, title1);
pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink);
new TypeReference<TextPageData<Asset>>() {
}, pageLink);
Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size());
@ -559,7 +576,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
pageLink = new TextPageLink(4, title2);
pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?",
new TypeReference<TextPageData<Asset>>(){}, pageLink);
new TypeReference<TextPageData<Asset>>() {
}, pageLink);
Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size());
}
@ -574,10 +592,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
String title1 = "Asset title 1";
String type1 = "typeC";
List<Asset> assetsType1 = new ArrayList<>();
for (int i=0;i<125;i++) {
for (int i = 0; i < 125; i++) {
Asset asset = new Asset();
String suffix = RandomStringUtils.randomAlphanumeric(15);
String name = title1+suffix;
String name = title1 + suffix;
name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase();
asset.setName(name);
asset.setType(type1);
@ -588,10 +606,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
String title2 = "Asset title 2";
String type2 = "typeD";
List<Asset> assetsType2 = new ArrayList<>();
for (int i=0;i<143;i++) {
for (int i = 0; i < 143; i++) {
Asset asset = new Asset();
String suffix = RandomStringUtils.randomAlphanumeric(15);
String name = title2+suffix;
String name = title2 + suffix;
name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase();
asset.setName(name);
asset.setType(type2);
@ -605,7 +623,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
TextPageData<Asset> pageData = null;
do {
pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&",
new TypeReference<TextPageData<Asset>>(){}, pageLink, type1);
new TypeReference<TextPageData<Asset>>() {
}, pageLink, type1);
loadedAssetsType1.addAll(pageData.getData());
if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink();
@ -621,7 +640,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
pageLink = new TextPageLink(4);
do {
pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&",
new TypeReference<TextPageData<Asset>>(){}, pageLink, type2);
new TypeReference<TextPageData<Asset>>() {
}, pageLink, type2);
loadedAssetsType2.addAll(pageData.getData());
if (pageData.hasNext()) {
pageLink = pageData.getNextPageLink();
@ -640,7 +660,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
pageLink = new TextPageLink(4);
pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&",
new TypeReference<TextPageData<Asset>>(){}, pageLink, type1);
new TypeReference<TextPageData<Asset>>() {
}, pageLink, type1);
Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size());
@ -651,7 +672,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest {
pageLink = new TextPageLink(4);
pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&",
new TypeReference<TextPageData<Asset>>(){}, pageLink, type2);
new TypeReference<TextPageData<Asset>>() {
}, pageLink, type2);
Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size());
}

6
application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java
View File

@ -426,7 +426,7 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes
message.setPayload(strKvs.getBytes());
client.publish("v1/devices/me/telemetry", message);
Thread.sleep(1000);
// client.disconnect();
client.disconnect();
}
private void awaitConnected(MqttAsyncClient client, long ms) throws InterruptedException {
@ -463,13 +463,13 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes
MqttConnectOptions options = new MqttConnectOptions();
options.setUserName(accessToken);
client.connect(options);
Thread.sleep(3000);
awaitConnected(client, TimeUnit.SECONDS.toMillis(30));
MqttMessage message = new MqttMessage();
message.setPayload((stringKV).getBytes());
client.publish("v1/devices/me/attributes", message);
Thread.sleep(1000);
client.disconnect();
return new HashSet<>(doGetAsync("/api/plugins/telemetry/DEVICE/" + viewDeviceId + "/keys/attributes", List.class));
}

7
application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java
View File

@ -16,10 +16,12 @@
package org.thingsboard.server.controller;
import org.cassandraunit.dataset.cql.ClassPathCQLDataSet;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.runner.RunWith;
import org.thingsboard.server.dao.CustomCassandraCQLUnit;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import java.util.Arrays;
@ -37,4 +39,9 @@ public class ControllerNoSqlTestSuite {
new ClassPathCQLDataSet("cassandra/system-data.cql", false, false),
new ClassPathCQLDataSet("cassandra/system-test.cql", false, false)),
"cassandra-test.yaml", 30000l);
@BeforeClass
public static void cleanupInMemStorage(){
InMemoryStorage.getInstance().cleanup();
}
}

7
application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java
View File

@ -15,10 +15,12 @@
*/
package org.thingsboard.server.controller;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.runner.RunWith;
import org.thingsboard.server.dao.CustomSqlUnit;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import java.util.Arrays;
@ -33,4 +35,9 @@ public class ControllerSqlTestSuite {
Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql"),
"sql/drop-all-tables.sql",
"sql-test.properties");
@BeforeClass
public static void cleanupInMemStorage(){
InMemoryStorage.getInstance().cleanup();
}
}

7
application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java
View File

@ -16,10 +16,12 @@
package org.thingsboard.server.mqtt;
import org.cassandraunit.dataset.cql.ClassPathCQLDataSet;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.runner.RunWith;
import org.thingsboard.server.dao.CustomCassandraCQLUnit;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import java.util.Arrays;
@ -36,4 +38,9 @@ public class MqttNoSqlTestSuite {
new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false),
new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)),
"cassandra-test.yaml", 30000l);
@BeforeClass
public static void cleanupInMemStorage(){
InMemoryStorage.getInstance().cleanup();
}
}

7
application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java
View File

@ -15,10 +15,12 @@
*/
package org.thingsboard.server.mqtt;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.runner.RunWith;
import org.thingsboard.server.dao.CustomSqlUnit;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import java.util.Arrays;
@ -32,4 +34,9 @@ public class MqttSqlTestSuite {
Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"),
"sql/drop-all-tables.sql",
"sql-test.properties");
@BeforeClass
public static void cleanupInMemStorage(){
InMemoryStorage.getInstance().cleanup();
}
}

19
application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java
View File

@ -37,6 +37,7 @@ import org.thingsboard.server.service.security.AccessValidator;
import java.util.Arrays;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@ -55,6 +56,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC
private User tenantAdmin;
private Long asyncContextTimeoutToUseRpcPlugin;
private static final AtomicInteger atomicInteger = new AtomicInteger(2);
@Before
public void beforeTest() throws Exception {
@ -70,7 +73,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC
tenantAdmin = new User();
tenantAdmin.setAuthority(Authority.TENANT_ADMIN);
tenantAdmin.setTenantId(savedTenant.getId());
tenantAdmin.setEmail("tenant2@thingsboard.org");
tenantAdmin.setEmail("tenant" + atomicInteger.getAndIncrement() + "@thingsboard.org");
tenantAdmin.setFirstName("Joe");
tenantAdmin.setLastName("Downs");
@ -109,6 +112,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC
client.subscribe("v1/devices/me/rpc/request/+", MqttQoS.AT_MOST_ONCE.value());
Thread.sleep(2000);
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}";
String deviceId = savedDevice.getId().getId().toString();
String result = doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isOk());
@ -128,7 +133,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC
String accessToken = deviceCredentials.getCredentialsId();
assertNotNull(accessToken);
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"timeout\": 6000}";
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"24\",\"value\": 1},\"timeout\": 6000}";
String deviceId = savedDevice.getId().getId().toString();
doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(),
@ -137,7 +142,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC
@Test
public void testServerMqttOneWayRpcDeviceDoesNotExist() throws Exception {
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}";
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"25\",\"value\": 1}}";
String nonExistentDeviceId = UUIDs.timeBased().toString();
String result = doPostAsync("/api/plugins/rpc/oneway/" + nonExistentDeviceId, setGpioRequest, String.class,
@ -165,7 +170,9 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC
client.subscribe("v1/devices/me/rpc/request/+", 1);
client.setCallback(new TestMqttCallback(client, new CountDownLatch(1)));
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}";
Thread.sleep(2000);
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"26\",\"value\": 1}}";
String deviceId = savedDevice.getId().getId().toString();
String result = doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isOk());
@ -183,7 +190,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC
String accessToken = deviceCredentials.getCredentialsId();
assertNotNull(accessToken);
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"timeout\": 6000}";
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"27\",\"value\": 1},\"timeout\": 6000}";
String deviceId = savedDevice.getId().getId().toString();
doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(),
@ -192,7 +199,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC
@Test
public void testServerMqttTwoWayRpcDeviceDoesNotExist() throws Exception {
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}";
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"28\",\"value\": 1}}";
String nonExistentDeviceId = UUIDs.timeBased().toString();
String result = doPostAsync("/api/plugins/rpc/twoway/" + nonExistentDeviceId, setGpioRequest, String.class,

11
application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java
View File

@ -79,7 +79,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr
String deviceId = savedDevice.getId().getId().toString();
Thread.sleep(1000);
Thread.sleep(2000);
List<String> actualKeys = doGetAsync("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", List.class);
Set<String> actualKeySet = new HashSet<>(actualKeys);
@ -97,7 +97,8 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr
assertEquals("4", values.get("key4").get(0).get("value"));
}
@Test
// @Test - Unstable
public void testMqttQoSLevel() throws Exception {
String clientId = MqttAsyncClient.generateClientId();
MqttAsyncClient client = new MqttAsyncClient(MQTT_URL, clientId);
@ -107,13 +108,13 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr
CountDownLatch latch = new CountDownLatch(1);
TestMqttCallback callback = new TestMqttCallback(client, latch);
client.setCallback(callback);
client.connect(options).waitForCompletion(3000);
client.connect(options).waitForCompletion(5000);
client.subscribe("v1/devices/me/attributes", MqttQoS.AT_MOST_ONCE.value());
String payload = "{\"key\":\"value\"}";
String payload = "{\"key\":\"uniqueValue\"}";
// TODO 3.1: we need to acknowledge subscription only after it is processed by device actor and not when the message is pushed to queue.
// MqttClient -> SUB REQUEST -> Transport -> Kafka -> Device Actor (subscribed)
// MqttClient <- SUB_ACK <- Transport
Thread.sleep(1000);
Thread.sleep(5000);
doPostAsync("/api/plugins/telemetry/" + savedDevice.getId() + "/SHARED_SCOPE", payload, String.class, status().isOk());
latch.await(10, TimeUnit.SECONDS);
assertEquals(payload, callback.getPayload());

7
application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java
View File

@ -16,11 +16,13 @@
package org.thingsboard.server.rules;
import org.cassandraunit.dataset.cql.ClassPathCQLDataSet;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.runner.RunWith;
import org.thingsboard.server.dao.CustomCassandraCQLUnit;
import org.thingsboard.server.dao.CustomSqlUnit;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import java.util.Arrays;
@ -40,4 +42,9 @@ public class RuleEngineNoSqlTestSuite {
new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)),
"cassandra-test.yaml", 30000l);
@BeforeClass
public static void cleanupInMemStorage(){
InMemoryStorage.getInstance().cleanup();
}
}

7
application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java
View File

@ -15,10 +15,12 @@
*/
package org.thingsboard.server.rules;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.runner.RunWith;
import org.thingsboard.server.dao.CustomSqlUnit;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import java.util.Arrays;
@ -33,4 +35,9 @@ public class RuleEngineSqlTestSuite {
Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"),
"sql/drop-all-tables.sql",
"sql-test.properties");
@BeforeClass
public static void cleanupInMemStorage(){
InMemoryStorage.getInstance().cleanup();
}
}

4
application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java
View File

@ -152,7 +152,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule
QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null);
// Pushing Message to the system
actorSystem.tell(qMsg, ActorRef.noSender());
Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess();
Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess();
TimePageData<Event> eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000);
List<Event> events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList());
@ -265,7 +265,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule
// Pushing Message to the system
actorSystem.tell(qMsg, ActorRef.noSender());
Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess();
Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess();
TimePageData<Event> eventsPage = getDebugEvents(savedTenant.getId(), rootRuleChain.getFirstRuleNodeId(), 1000);
List<Event> events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList());

21
application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java
View File

@ -17,6 +17,7 @@ package org.thingsboard.server.service.cluster.routing;
import com.datastax.driver.core.utils.UUIDs;
import lombok.extern.slf4j.Slf4j;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -31,6 +32,8 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.discovery.TenantRoutingInfoService;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import java.util.ArrayList;
import java.util.Collections;
@ -41,6 +44,7 @@ import java.util.Map;
import java.util.stream.Collectors;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@Slf4j
@RunWith(MockitoJUnitRunner.class)
@ -52,6 +56,7 @@ public class ConsistentHashParitionServiceTest {
private TbServiceInfoProvider discoveryService;
private TenantRoutingInfoService routingInfoService;
private ApplicationEventPublisher applicationEventPublisher;
private TbQueueRuleEngineSettings ruleEngineSettings;
private String hashFunctionName = "murmur3_128";
private Integer virtualNodesSize = 16;
@ -62,12 +67,15 @@ public class ConsistentHashParitionServiceTest {
discoveryService = mock(TbServiceInfoProvider.class);
applicationEventPublisher = mock(ApplicationEventPublisher.class);
routingInfoService = mock(TenantRoutingInfoService.class);
clusterRoutingService = new ConsistentHashPartitionService(discoveryService, routingInfoService, applicationEventPublisher);
ruleEngineSettings = mock(TbQueueRuleEngineSettings.class);
clusterRoutingService = new ConsistentHashPartitionService(discoveryService,
routingInfoService,
applicationEventPublisher,
ruleEngineSettings
);
when(ruleEngineSettings.getQueues()).thenReturn(Collections.emptyList());
ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core");
ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 3);
ReflectionTestUtils.setField(clusterRoutingService, "ruleEngineTopic", "tb.rule-engine");
ReflectionTestUtils.setField(clusterRoutingService, "ruleEnginePartitions", 100);
ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName);
ReflectionTestUtils.setField(clusterRoutingService, "virtualNodesSize", virtualNodesSize);
TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder()
@ -107,8 +115,9 @@ public class ConsistentHashParitionServiceTest {
List<Map.Entry<Integer, Integer>> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList());
long end = System.currentTimeMillis();
double diff = (data.get(data.size() - 1).getValue() - data.get(0).getValue());
System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", (diff / ITERATIONS) * 100.0) + "%)");
double diffPercent = (diff / ITERATIONS) * 100.0;
System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", diffPercent) + "%)");
Assert.assertTrue(diffPercent < 0.5);
for (Map.Entry<Integer, Integer> entry : data) {
System.out.println(entry.getKey() + ": " + entry.getValue());
}

66
application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java
View File

@ -0,0 +1,66 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.queue;
import lombok.extern.slf4j.Slf4j;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.mockito.runners.MockitoJUnitRunner;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@Slf4j
@RunWith(MockitoJUnitRunner.class)
public class TbMsgPackProcessingContextTest {
@Test
public void testHighConcurrencyCase() throws InterruptedException {
TbRuleEngineSubmitStrategy strategyMock = mock(TbRuleEngineSubmitStrategy.class);
int msgCount = 1000;
int parallelCount = 5;
ExecutorService executorService = Executors.newFixedThreadPool(parallelCount);
try {
ConcurrentMap<UUID, TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> messages = new ConcurrentHashMap<>();
for (int i = 0; i < msgCount; i++) {
messages.put(UUID.randomUUID(), new TbProtoQueueMsg<>(UUID.randomUUID(), null));
}
when(strategyMock.getPendingMap()).thenReturn(messages);
TbMsgPackProcessingContext context = new TbMsgPackProcessingContext(strategyMock);
for (UUID uuid : messages.keySet()) {
for (int i = 0; i < parallelCount; i++) {
executorService.submit(() -> context.onSuccess(uuid));
}
}
Assert.assertTrue(context.await(10, TimeUnit.SECONDS));
Mockito.verify(strategyMock, Mockito.times(msgCount)).onSuccess(Mockito.any(UUID.class));
} finally {
executorService.shutdownNow();
}
}
}

7
application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java
View File

@ -16,10 +16,12 @@
package org.thingsboard.server.system;
import org.cassandraunit.dataset.cql.ClassPathCQLDataSet;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.runner.RunWith;
import org.thingsboard.server.dao.CustomCassandraCQLUnit;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import java.util.Arrays;
@ -38,4 +40,9 @@ public class SystemNoSqlTestSuite {
new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false),
new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)),
"cassandra-test.yaml", 30000l);
@BeforeClass
public static void cleanupInMemStorage(){
InMemoryStorage.getInstance().cleanup();
}
}

7
application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java
View File

@ -15,10 +15,12 @@
*/
package org.thingsboard.server.system;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.runner.RunWith;
import org.thingsboard.server.dao.CustomSqlUnit;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import java.util.Arrays;
@ -35,4 +37,9 @@ public class SystemSqlTestSuite {
"sql/drop-all-tables.sql",
"sql-test.properties");
@BeforeClass
public static void cleanupInMemStorage(){
InMemoryStorage.getInstance().cleanup();
}
}

38
common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java
View File

@ -16,27 +16,32 @@
package org.thingsboard.server.queue.azure.servicebus;
import com.microsoft.azure.servicebus.management.ManagementClient;
import com.microsoft.azure.servicebus.management.QueueDescription;
import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder;
import com.microsoft.azure.servicebus.primitives.ServiceBusException;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Component;
import org.thingsboard.server.queue.TbQueueAdmin;
import java.io.IOException;
import java.time.Duration;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@Slf4j
@Component
@ConditionalOnExpression("'${queue.type:null}'=='service-bus'")
public class TbServiceBusAdmin implements TbQueueAdmin {
private final String MAX_SIZE = "maxSizeInMb";
private final String MESSAGE_TIME_TO_LIVE = "messageTimeToLiveInSec";
private final String LOCK_DURATION = "lockDurationInSec";
private final Map<String, String> queueConfigs;
private final Set<String> queues = ConcurrentHashMap.newKeySet();
private final ManagementClient client;
public TbServiceBusAdmin(TbServiceBusSettings serviceBusSettings) {
public TbServiceBusAdmin(TbServiceBusSettings serviceBusSettings, Map<String, String> queueConfigs) {
this.queueConfigs = queueConfigs;
ConnectionStringBuilder builder = new ConnectionStringBuilder(
serviceBusSettings.getNamespaceName(),
"queues",
@ -60,13 +65,34 @@ public class TbServiceBusAdmin implements TbQueueAdmin {
}
try {
client.createQueue(topic);
QueueDescription queueDescription = new QueueDescription(topic);
setQueueConfigs(queueDescription);
client.createQueue(queueDescription);
queues.add(topic);
} catch (ServiceBusException | InterruptedException e) {
log.error("Failed to create queue: [{}]", topic, e);
}
}
private void setQueueConfigs(QueueDescription queueDescription) {
queueConfigs.forEach((confKey, confValue) -> {
switch (confKey) {
case MAX_SIZE:
queueDescription.setMaxSizeInMB(Long.parseLong(confValue));
break;
case MESSAGE_TIME_TO_LIVE:
queueDescription.setDefaultMessageTimeToLive(Duration.ofSeconds(Long.parseLong(confValue)));
break;
case LOCK_DURATION:
queueDescription.setLockDuration(Duration.ofSeconds(Long.parseLong(confValue)));
break;
default:
log.error("Unknown config: [{}]", confKey);
}
});
}
public void destroy() {
try {
client.close();

71
common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java
View File

@ -0,0 +1,71 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.azure.servicebus;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.HashMap;
import java.util.Map;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='service-bus'")
public class TbServiceBusQueueConfigs {
@Value("${queue.service-bus.queue-properties.core}")
private String coreProperties;
@Value("${queue.service-bus.queue-properties.rule-engine}")
private String ruleEngineProperties;
@Value("${queue.service-bus.queue-properties.transport-api}")
private String transportApiProperties;
@Value("${queue.service-bus.queue-properties.notifications}")
private String notificationsProperties;
@Value("${queue.service-bus.queue-properties.js-executor}")
private String jsExecutorProperties;
@Getter
private Map<String, String> coreConfigs;
@Getter
private Map<String, String> ruleEngineConfigs;
@Getter
private Map<String, String> transportApiConfigs;
@Getter
private Map<String, String> notificationsConfigs;
@Getter
private Map<String, String> jsExecutorConfigs;
@PostConstruct
private void init() {
coreConfigs = getConfigs(coreProperties);
ruleEngineConfigs = getConfigs(ruleEngineProperties);
transportApiConfigs = getConfigs(transportApiProperties);
notificationsConfigs = getConfigs(notificationsProperties);
jsExecutorConfigs = getConfigs(jsExecutorProperties);
}
private Map<String, String> getConfigs(String properties) {
Map<String, String> configs = new HashMap<>();
for (String property : properties.split(";")) {
int delimiterPosition = property.indexOf(":");
String key = property.substring(0, delimiterPosition);
String value = property.substring(delimiterPosition + 1);
configs.put(key, value);
}
return configs;
}
}

12
common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java
View File

@ -30,6 +30,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import javax.annotation.PostConstruct;
import java.nio.charset.StandardCharsets;
@ -61,6 +62,7 @@ public class ConsistentHashPartitionService implements PartitionService {
private final ApplicationEventPublisher applicationEventPublisher;
private final TbServiceInfoProvider serviceInfoProvider;
private final TenantRoutingInfoService tenantRoutingInfoService;
private final TbQueueRuleEngineSettings tbQueueRuleEngineSettings;
private final ConcurrentMap<ServiceQueue, String> partitionTopics = new ConcurrentHashMap<>();
private final ConcurrentMap<ServiceQueue, Integer> partitionSizes = new ConcurrentHashMap<>();
private final ConcurrentMap<TenantId, TenantRoutingInfo> tenantRoutingInfoMap = new ConcurrentHashMap<>();
@ -74,10 +76,14 @@ public class ConsistentHashPartitionService implements PartitionService {
private HashFunction hashFunction;
public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider, TenantRoutingInfoService tenantRoutingInfoService, ApplicationEventPublisher applicationEventPublisher) {
public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider,
TenantRoutingInfoService tenantRoutingInfoService,
ApplicationEventPublisher applicationEventPublisher,
TbQueueRuleEngineSettings tbQueueRuleEngineSettings) {
this.serviceInfoProvider = serviceInfoProvider;
this.tenantRoutingInfoService = tenantRoutingInfoService;
this.applicationEventPublisher = applicationEventPublisher;
this.tbQueueRuleEngineSettings = tbQueueRuleEngineSettings;
}
@PostConstruct
@ -85,6 +91,10 @@ public class ConsistentHashPartitionService implements PartitionService {
this.hashFunction = forName(hashFunctionName);
partitionSizes.put(new ServiceQueue(ServiceType.TB_CORE), corePartitions);
partitionTopics.put(new ServiceQueue(ServiceType.TB_CORE), coreTopic);
tbQueueRuleEngineSettings.getQueues().forEach(queueConfiguration -> {
partitionTopics.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queueConfiguration.getName()), queueConfiguration.getTopic());
partitionSizes.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queueConfiguration.getName()), queueConfiguration.getPartitions());
});
}
@Override

42
common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java
View File

@ -30,11 +30,9 @@ import java.util.concurrent.TimeUnit;
public final class InMemoryStorage {
private static InMemoryStorage instance;
private final ConcurrentHashMap<String, BlockingQueue<TbQueueMsg>> storage;
private volatile boolean stopped;
private InMemoryStorage() {
storage = new ConcurrentHashMap<>();
stopped = false;
}
public static InMemoryStorage getInstance() {
@ -52,33 +50,31 @@ public final class InMemoryStorage {
return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg);
}
public <T extends TbQueueMsg> List<T> get(String topic, long durationInMillis) {
public <T extends TbQueueMsg> List<T> get(String topic) throws InterruptedException {
if (storage.containsKey(topic)) {
try {
List<T> entities;
T first = (T) storage.get(topic).poll(durationInMillis, TimeUnit.MILLISECONDS);
if (first != null) {
entities = new ArrayList<>();
entities.add(first);
List<TbQueueMsg> otherList = new ArrayList<>();
storage.get(topic).drainTo(otherList, 999);
for (TbQueueMsg other : otherList) {
entities.add((T) other);
}
} else {
entities = Collections.emptyList();
}
return entities;
} catch (InterruptedException e) {
if (!stopped) {
log.warn("Queue was interrupted", e);
List<T> entities;
T first = (T) storage.get(topic).poll();
if (first != null) {
entities = new ArrayList<>();
entities.add(first);
List<TbQueueMsg> otherList = new ArrayList<>();
storage.get(topic).drainTo(otherList, 999);
for (TbQueueMsg other : otherList) {
entities.add((T) other);
}
} else {
entities = Collections.emptyList();
}
return entities;
}
return Collections.emptyList();
}
public void stop() {
stopped = true;
/**
* Used primarily for testing.
*/
public void cleanup() {
storage.clear();
}
}

11
common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java
View File

@ -66,7 +66,16 @@ public class InMemoryTbQueueConsumer<T extends TbQueueMsg> implements TbQueueCon
if (subscribed) {
List<T> messages = partitions
.stream()
.map(tpi -> storage.get(tpi.getFullTopicName(), durationInMillis))
.map(tpi -> {
try {
return storage.get(tpi.getFullTopicName());
} catch (InterruptedException e) {
if (!stopped) {
log.error("Queue was interrupted.", e);
}
return Collections.emptyList();
}
})
.flatMap(List::stream)
.map(msg -> (T) msg).collect(Collectors.toList());
if (messages.size() > 0) {

11
common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java
View File

@ -30,6 +30,7 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin;
@ -47,8 +48,10 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory {
private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbAwsSqsSettings sqsSettings;
private final TbQueueCoreSettings coreSettings;
private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
@ -56,12 +59,15 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory {
TbQueueTransportNotificationSettings transportNotificationSettings,
TbAwsSqsSettings sqsSettings,
TbServiceInfoProvider serviceInfoProvider,
TbQueueCoreSettings coreSettings,
TbAwsSqsQueueAttributes sqsQueueAttributes) {
this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings;
this.sqsSettings = sqsSettings;
this.serviceInfoProvider = serviceInfoProvider;
this.coreSettings = coreSettings;
this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes());
this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes());
this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes());
}
@ -94,7 +100,7 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory {
@Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic());
return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic());
}
@Override
@ -105,6 +111,9 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory {
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}

9
common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java
View File

@ -32,9 +32,9 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer;
import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')")
@ -43,13 +43,16 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory
private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueCoreSettings coreSettings;
public InMemoryTbTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings,
TbQueueTransportNotificationSettings transportNotificationSettings,
TbServiceInfoProvider serviceInfoProvider) {
TbServiceInfoProvider serviceInfoProvider,
TbQueueCoreSettings coreSettings) {
this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings;
this.serviceInfoProvider = serviceInfoProvider;
this.coreSettings = coreSettings;
}
@Override
@ -86,7 +89,7 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory
@Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic());
return new InMemoryTbQueueProducer<>(coreSettings.getTopic());
}
@Override

20
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java
View File

@ -49,6 +49,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import javax.annotation.PreDestroy;
import java.nio.charset.StandardCharsets;
@Component
@ -247,4 +248,23 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
return builder.build();
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
}

19
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java
View File

@ -47,6 +47,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import javax.annotation.PreDestroy;
import java.nio.charset.StandardCharsets;
@Component
@ -218,4 +219,22 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
return builder.build();
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
}

17
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java
View File

@ -45,6 +45,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import javax.annotation.PreDestroy;
import java.nio.charset.StandardCharsets;
@Component
@ -190,4 +191,20 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
return builder.build();
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
}

18
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java
View File

@ -40,6 +40,8 @@ import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import javax.annotation.PreDestroy;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')")
@Slf4j
@ -135,4 +137,20 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory {
responseBuilder.admin(notificationAdmin);
return responseBuilder.build();
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
}

64
common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java
View File

@ -42,6 +42,7 @@ import org.thingsboard.server.queue.pubsub.TbPubSubAdmin;
import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate;
import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate;
import org.thingsboard.server.queue.pubsub.TbPubSubSettings;
import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
@ -49,6 +50,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import javax.annotation.PreDestroy;
import java.nio.charset.StandardCharsets;
@Component
@ -60,11 +62,16 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
private final TbQueueRuleEngineSettings ruleEngineSettings;
private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbQueueAdmin admin;
private final PartitionService partitionService;
private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin jsExecutorAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings,
TbQueueCoreSettings coreSettings,
TbQueueRuleEngineSettings ruleEngineSettings,
@ -72,86 +79,92 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
TbQueueTransportNotificationSettings transportNotificationSettings,
PartitionService partitionService,
TbServiceInfoProvider serviceInfoProvider,
TbPubSubSubscriptionSettings pubSubSubscriptionSettings,
TbQueueRemoteJsInvokeSettings jsInvokeSettings) {
this.pubSubSettings = pubSubSettings;
this.coreSettings = coreSettings;
this.ruleEngineSettings = ruleEngineSettings;
this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings;
this.admin = new TbPubSubAdmin(pubSubSettings);
this.partitionService = partitionService;
this.serviceInfoProvider = serviceInfoProvider;
this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings());
this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings());
this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings());
this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings());
this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings());
this.jsInvokeSettings = jsInvokeSettings;
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportNotificationSettings.getNotificationsTopic());
return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic());
return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic());
return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic());
return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic());
return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic(),
return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings,
return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings,
partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, coreSettings.getTopic(),
return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings,
return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings,
partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic(),
return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getResponsesTopic());
return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic());
}
@Override
@Bean
public TbQueueRequestTemplate<TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> createRemoteJsRequestTemplate() {
TbQueueProducer<TbProtoJsQueueMsg<RemoteJsRequest>> producer = new TbPubSubProducerTemplate<>(admin, pubSubSettings, jsInvokeSettings.getRequestTopic());
TbQueueConsumer<TbProtoQueueMsg<RemoteJsResponse>> consumer = new TbPubSubConsumerTemplate<>(admin, pubSubSettings,
TbQueueProducer<TbProtoJsQueueMsg<RemoteJsRequest>> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic());
TbQueueConsumer<TbProtoQueueMsg<RemoteJsResponse>> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings,
jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(),
msg -> {
RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder();
@ -161,7 +174,7 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
<TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder();
builder.queueAdmin(admin);
builder.queueAdmin(jsExecutorAdmin);
builder.requestTemplate(producer);
builder.responseTemplate(consumer);
builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests());
@ -169,4 +182,23 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
return builder.build();
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
}

54
common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java
View File

@ -30,6 +30,8 @@ import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.pubsub.TbPubSubAdmin;
import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
@ -40,6 +42,8 @@ import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate;
import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate;
import org.thingsboard.server.queue.pubsub.TbPubSubSettings;
import javax.annotation.PreDestroy;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'")
public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory {
@ -47,75 +51,99 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory {
private final TbPubSubSettings pubSubSettings;
private final TbQueueCoreSettings coreSettings;
private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueAdmin admin;
private final PartitionService partitionService;
private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin jsExecutorAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
public PubSubTbCoreQueueFactory(TbPubSubSettings pubSubSettings,
TbQueueCoreSettings coreSettings,
TbQueueTransportApiSettings transportApiSettings,
TbQueueAdmin admin,
PartitionService partitionService,
TbServiceInfoProvider serviceInfoProvider) {
TbServiceInfoProvider serviceInfoProvider,
TbPubSubSubscriptionSettings pubSubSubscriptionSettings) {
this.pubSubSettings = pubSubSettings;
this.coreSettings = coreSettings;
this.transportApiSettings = transportApiSettings;
this.admin = admin;
this.partitionService = partitionService;
this.serviceInfoProvider = serviceInfoProvider;
this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings());
this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings());
this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings());
this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic());
return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic());
return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic());
return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic());
return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic());
return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, coreSettings.getTopic(),
return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings,
return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings,
partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic(),
return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic());
return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
}
@Override
public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
return null;
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
}

50
common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java
View File

@ -32,13 +32,17 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.pubsub.TbPubSubAdmin;
import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate;
import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate;
import org.thingsboard.server.queue.pubsub.TbPubSubSettings;
import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import javax.annotation.PreDestroy;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'")
public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
@ -46,59 +50,67 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory
private final TbPubSubSettings pubSubSettings;
private final TbQueueCoreSettings coreSettings;
private final TbQueueRuleEngineSettings ruleEngineSettings;
private final TbQueueAdmin admin;
private final PartitionService partitionService;
private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin jsExecutorAdmin;
private final TbQueueAdmin notificationAdmin;
public PubSubTbRuleEngineQueueFactory(TbPubSubSettings pubSubSettings,
TbQueueCoreSettings coreSettings,
TbQueueRuleEngineSettings ruleEngineSettings,
TbQueueAdmin admin,
PartitionService partitionService,
TbServiceInfoProvider serviceInfoProvider) {
TbServiceInfoProvider serviceInfoProvider,
TbPubSubSubscriptionSettings pubSubSubscriptionSettings) {
this.pubSubSettings = pubSubSettings;
this.coreSettings = coreSettings;
this.ruleEngineSettings = ruleEngineSettings;
this.admin = admin;
this.partitionService = partitionService;
this.serviceInfoProvider = serviceInfoProvider;
this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings());
this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings());
this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings());
this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic());
return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic());
return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic());
return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic());
return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic());
return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic(),
return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings,
return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings,
partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@ -107,4 +119,20 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory
public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
return null;
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
}

54
common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java
View File

@ -25,12 +25,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
@ -38,6 +34,13 @@ import org.thingsboard.server.queue.pubsub.TbPubSubAdmin;
import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate;
import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate;
import org.thingsboard.server.queue.pubsub.TbPubSubSettings;
import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import javax.annotation.PreDestroy;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')")
@ -50,33 +53,42 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory {
private final TbQueueRuleEngineSettings ruleEngineSettings;
private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbQueueAdmin admin;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
public PubSubTransportQueueFactory(TbPubSubSettings pubSubSettings,
TbServiceInfoProvider serviceInfoProvider,
TbQueueCoreSettings coreSettings,
TbQueueRuleEngineSettings ruleEngineSettings,
TbQueueTransportApiSettings transportApiSettings,
TbQueueTransportNotificationSettings transportNotificationSettings) {
TbQueueTransportNotificationSettings transportNotificationSettings,
TbPubSubSubscriptionSettings pubSubSubscriptionSettings) {
this.pubSubSettings = pubSubSettings;
this.serviceInfoProvider = serviceInfoProvider;
this.coreSettings = coreSettings;
this.ruleEngineSettings = ruleEngineSettings;
this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings;
this.admin = new TbPubSubAdmin(pubSubSettings);
this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings());
this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings());
this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings());
this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings());
}
@Override
public TbQueueRequestTemplate<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiRequestTemplate() {
TbQueueProducer<TbProtoQueueMsg<TransportApiRequestMsg>> producer = new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic());
TbQueueConsumer<TbProtoQueueMsg<TransportApiResponseMsg>> consumer = new TbPubSubConsumerTemplate<>(admin, pubSubSettings,
TbQueueProducer<TbProtoQueueMsg<TransportApiRequestMsg>> producer = new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic());
TbQueueConsumer<TbProtoQueueMsg<TransportApiResponseMsg>> consumer = new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings,
transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders()));
DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> templateBuilder = DefaultTbQueueRequestTemplate.builder();
templateBuilder.queueAdmin(admin);
templateBuilder.queueAdmin(transportApiAdmin);
templateBuilder.requestTemplate(producer);
templateBuilder.responseTemplate(consumer);
templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests());
@ -87,18 +99,34 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory {
@Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic());
return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic());
return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic());
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsConsumer() {
return new TbPubSubConsumerTemplate<>(admin, pubSubSettings,
return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings,
transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
}

71
common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java
View File

@ -32,10 +32,10 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate;
import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
@ -44,6 +44,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import javax.annotation.PreDestroy;
import java.nio.charset.StandardCharsets;
@Component
@ -57,9 +58,13 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbRabbitMqSettings rabbitMqSettings;
private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
private final TbQueueAdmin admin;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin jsExecutorAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
public RabbitMqMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings,
TbQueueRuleEngineSettings ruleEngineSettings,
@ -67,8 +72,8 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
TbQueueTransportApiSettings transportApiSettings,
TbQueueTransportNotificationSettings transportNotificationSettings,
TbRabbitMqSettings rabbitMqSettings,
TbQueueRemoteJsInvokeSettings jsInvokeSettings,
TbQueueAdmin admin) {
TbRabbitMqQueueArguments queueArguments,
TbQueueRemoteJsInvokeSettings jsInvokeSettings) {
this.partitionService = partitionService;
this.coreSettings = coreSettings;
this.serviceInfoProvider = serviceInfoProvider;
@ -76,77 +81,82 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings;
this.rabbitMqSettings = rabbitMqSettings;
this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs());
this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs());
this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs());
this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs());
this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs());
this.jsInvokeSettings = jsInvokeSettings;
this.admin = admin;
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic());
return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic());
return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic());
return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> createTbCoreMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic());
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic());
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic(),
return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings,
return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings,
partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> createToCoreMsgConsumer() {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic(),
return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings,
return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings,
partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>> createTransportApiRequestConsumer() {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic(),
return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> createTransportApiResponseProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getResponsesTopic());
return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getResponsesTopic());
}
@Override
@Bean
public TbQueueRequestTemplate<TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> createRemoteJsRequestTemplate() {
TbQueueProducer<TbProtoJsQueueMsg<RemoteJsRequest>> producer = new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, jsInvokeSettings.getRequestTopic());
TbQueueConsumer<TbProtoQueueMsg<RemoteJsResponse>> consumer = new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings,
TbQueueProducer<TbProtoJsQueueMsg<RemoteJsRequest>> producer = new TbRabbitMqProducerTemplate<>(jsExecutorAdmin, rabbitMqSettings, jsInvokeSettings.getRequestTopic());
TbQueueConsumer<TbProtoQueueMsg<RemoteJsResponse>> consumer = new TbRabbitMqConsumerTemplate<>(jsExecutorAdmin, rabbitMqSettings,
jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(),
msg -> {
RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder();
@ -156,7 +166,7 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
<TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder();
builder.queueAdmin(admin);
builder.queueAdmin(jsExecutorAdmin);
builder.requestTemplate(producer);
builder.responseTemplate(consumer);
builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests());
@ -164,4 +174,23 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
return builder.build();
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
}

57
common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java
View File

@ -34,13 +34,17 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import javax.annotation.PreDestroy;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-core'")
public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory {
@ -51,7 +55,12 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory {
private final TbQueueTransportApiSettings transportApiSettings;
private final PartitionService partitionService;
private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueAdmin admin;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin jsExecutorAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
public RabbitMqTbCoreQueueFactory(TbRabbitMqSettings rabbitMqSettings,
TbQueueCoreSettings coreSettings,
@ -59,67 +68,91 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory {
TbQueueRuleEngineSettings ruleEngineSettings,
PartitionService partitionService,
TbServiceInfoProvider serviceInfoProvider,
TbQueueAdmin admin) {
TbRabbitMqQueueArguments queueArguments) {
this.rabbitMqSettings = rabbitMqSettings;
this.coreSettings = coreSettings;
this.transportApiSettings = transportApiSettings;
this.ruleEngineSettings = ruleEngineSettings;
this.partitionService = partitionService;
this.serviceInfoProvider = serviceInfoProvider;
this.admin = admin;
this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs());
this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs());
this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs());
this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs());
this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic());
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic());
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic());
return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic());
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic());
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic(),
return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings,
return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings,
partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic(),
return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic());
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
}
@Override
public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
return null;
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
}

48
common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java
View File

@ -32,13 +32,17 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import javax.annotation.PreDestroy;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-rule-engine'")
public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
@ -48,55 +52,63 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor
private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueRuleEngineSettings ruleEngineSettings;
private final TbRabbitMqSettings rabbitMqSettings;
private final TbQueueAdmin admin;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin jsExecutorAdmin;
private final TbQueueAdmin notificationAdmin;
public RabbitMqTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings,
TbQueueRuleEngineSettings ruleEngineSettings,
TbServiceInfoProvider serviceInfoProvider,
TbRabbitMqSettings rabbitMqSettings,
TbQueueAdmin admin) {
TbRabbitMqQueueArguments queueArguments) {
this.partitionService = partitionService;
this.coreSettings = coreSettings;
this.serviceInfoProvider = serviceInfoProvider;
this.ruleEngineSettings = ruleEngineSettings;
this.rabbitMqSettings = rabbitMqSettings;
this.admin = admin;
this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs());
this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs());
this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs());
this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic());
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic());
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic());
return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic());
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic());
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic(),
return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings,
return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings,
partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@ -105,4 +117,20 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor
public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
return null;
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
}

50
common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java
View File

@ -30,12 +30,17 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import javax.annotation.PreDestroy;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')")
@Slf4j
@ -43,34 +48,45 @@ public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory {
private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbRabbitMqSettings rabbitMqSettings;
private final TbQueueAdmin admin;
private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueCoreSettings coreSettings;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
public RabbitMqTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings,
TbQueueTransportNotificationSettings transportNotificationSettings,
TbRabbitMqSettings rabbitMqSettings,
TbServiceInfoProvider serviceInfoProvider,
TbQueueAdmin admin) {
TbQueueCoreSettings coreSettings,
TbRabbitMqQueueArguments queueArguments) {
this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings;
this.rabbitMqSettings = rabbitMqSettings;
this.admin = admin;
this.serviceInfoProvider = serviceInfoProvider;
this.coreSettings = coreSettings;
this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs());
this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs());
this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs());
this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs());
}
@Override
public TbQueueRequestTemplate<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiRequestTemplate() {
TbRabbitMqProducerTemplate<TbProtoQueueMsg<TransportApiRequestMsg>> producerTemplate =
new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic());
new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic());
TbRabbitMqConsumerTemplate<TbProtoQueueMsg<TransportApiResponseMsg>> consumerTemplate =
new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings,
new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings,
transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders()));
DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> templateBuilder = DefaultTbQueueRequestTemplate.builder();
templateBuilder.queueAdmin(admin);
templateBuilder.queueAdmin(transportApiAdmin);
templateBuilder.requestTemplate(producerTemplate);
templateBuilder.responseTemplate(consumerTemplate);
templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests());
@ -81,17 +97,33 @@ public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory {
@Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic());
return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic());
return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic());
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsConsumer() {
return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(),
return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
}

63
common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java
View File

@ -30,19 +30,25 @@ import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings;
import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.kafka.TbKafkaAdmin;
import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import javax.annotation.PreDestroy;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'")
public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory {
@ -54,7 +60,12 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul
private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbServiceBusSettings serviceBusSettings;
private final TbQueueAdmin admin;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin jsExecutorAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
public ServiceBusMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings,
TbQueueRuleEngineSettings ruleEngineSettings,
@ -62,7 +73,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul
TbQueueTransportApiSettings transportApiSettings,
TbQueueTransportNotificationSettings transportNotificationSettings,
TbServiceBusSettings serviceBusSettings,
TbQueueAdmin admin) {
TbServiceBusQueueConfigs serviceBusQueueConfigs) {
this.partitionService = partitionService;
this.coreSettings = coreSettings;
this.serviceInfoProvider = serviceInfoProvider;
@ -70,73 +81,97 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul
this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings;
this.serviceBusSettings = serviceBusSettings;
this.admin = admin;
this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs());
this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs());
this.jsExecutorAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getJsExecutorConfigs());
this.transportApiAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getTransportApiConfigs());
this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic());
return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic());
return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic());
return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic());
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic());
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(),
return new TbServiceBusConsumerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings,
return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings,
partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(),
return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings,
return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings,
partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(),
return new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getResponsesTopic());
return new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getResponsesTopic());
}
@Override
public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
return null;
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
}

57
common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java
View File

@ -29,8 +29,10 @@ import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings;
import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -40,6 +42,8 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import javax.annotation.PreDestroy;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'")
public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory {
@ -50,7 +54,12 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory {
private final TbQueueTransportApiSettings transportApiSettings;
private final PartitionService partitionService;
private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueAdmin admin;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin jsExecutorAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
public ServiceBusTbCoreQueueFactory(TbServiceBusSettings serviceBusSettings,
TbQueueCoreSettings coreSettings,
@ -58,67 +67,91 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory {
TbQueueRuleEngineSettings ruleEngineSettings,
PartitionService partitionService,
TbServiceInfoProvider serviceInfoProvider,
TbQueueAdmin admin) {
TbServiceBusQueueConfigs serviceBusQueueConfigs) {
this.serviceBusSettings = serviceBusSettings;
this.coreSettings = coreSettings;
this.transportApiSettings = transportApiSettings;
this.ruleEngineSettings = ruleEngineSettings;
this.partitionService = partitionService;
this.serviceInfoProvider = serviceInfoProvider;
this.admin = admin;
this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs());
this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs());
this.jsExecutorAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getJsExecutorConfigs());
this.transportApiAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getTransportApiConfigs());
this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic());
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic());
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic());
return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic());
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic());
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(),
return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings,
return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings,
partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(),
return new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic());
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
}
@Override
public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
return null;
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
}

48
common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java
View File

@ -27,8 +27,10 @@ import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings;
import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -38,6 +40,8 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import javax.annotation.PreDestroy;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-rule-engine'")
public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
@ -47,55 +51,63 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact
private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueRuleEngineSettings ruleEngineSettings;
private final TbServiceBusSettings serviceBusSettings;
private final TbQueueAdmin admin;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
private final TbQueueAdmin jsExecutorAdmin;
private final TbQueueAdmin notificationAdmin;
public ServiceBusTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings,
TbQueueRuleEngineSettings ruleEngineSettings,
TbServiceInfoProvider serviceInfoProvider,
TbServiceBusSettings serviceBusSettings,
TbQueueAdmin admin) {
TbServiceBusQueueConfigs serviceBusQueueConfigs) {
this.partitionService = partitionService;
this.coreSettings = coreSettings;
this.serviceInfoProvider = serviceInfoProvider;
this.ruleEngineSettings = ruleEngineSettings;
this.serviceBusSettings = serviceBusSettings;
this.admin = admin;
this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs());
this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs());
this.jsExecutorAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getJsExecutorConfigs());
this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic());
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic());
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic());
return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic());
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic());
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(),
return new TbServiceBusConsumerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings,
return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings,
partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@ -104,4 +116,20 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact
public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
return null;
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (ruleEngineAdmin != null) {
ruleEngineAdmin.destroy();
}
if (jsExecutorAdmin != null) {
jsExecutorAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
}

51
common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java
View File

@ -23,15 +23,20 @@ import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs;
import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings;
import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
import javax.annotation.PreDestroy;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')")
@Slf4j
@ -39,34 +44,43 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory
private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbServiceBusSettings serviceBusSettings;
private final TbQueueAdmin admin;
private final TbServiceInfoProvider serviceInfoProvider;
private final TbQueueCoreSettings coreSettings;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin transportApiAdmin;
private final TbQueueAdmin notificationAdmin;
public ServiceBusTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings,
TbQueueTransportNotificationSettings transportNotificationSettings,
TbServiceBusSettings serviceBusSettings,
TbServiceInfoProvider serviceInfoProvider,
TbQueueAdmin admin) {
TbQueueTransportNotificationSettings transportNotificationSettings,
TbServiceBusSettings serviceBusSettings,
TbServiceInfoProvider serviceInfoProvider,
TbQueueCoreSettings coreSettings,
TbServiceBusQueueConfigs serviceBusQueueConfigs) {
this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings;
this.serviceBusSettings = serviceBusSettings;
this.admin = admin;
this.serviceInfoProvider = serviceInfoProvider;
this.coreSettings = coreSettings;
this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs());
this.transportApiAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getTransportApiConfigs());
this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs());
}
@Override
public TbQueueRequestTemplate<TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>, TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> createTransportApiRequestTemplate() {
TbQueueProducer<TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>> producerTemplate =
new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic());
new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic());
TbQueueConsumer<TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> consumerTemplate =
new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings,
new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings,
transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders()));
DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
<TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>, TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> templateBuilder = DefaultTbQueueRequestTemplate.builder();
templateBuilder.queueAdmin(admin);
templateBuilder.queueAdmin(transportApiAdmin);
templateBuilder.requestTemplate(producerTemplate);
templateBuilder.responseTemplate(consumerTemplate);
templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests());
@ -77,18 +91,31 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> createRuleEngineMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic());
return new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic());
}
@Override
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> createTbCoreMsgProducer() {
return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic());
return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic());
}
@Override
public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToTransportMsg>> createTransportNotificationsConsumer() {
return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings,
return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings,
transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(),
msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders()));
}
@PreDestroy
private void destroy() {
if (coreAdmin != null) {
coreAdmin.destroy();
}
if (transportApiAdmin != null) {
transportApiAdmin.destroy();
}
if (notificationAdmin != null) {
notificationAdmin.destroy();
}
}
}

120
common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java
View File

@ -19,33 +19,40 @@ import com.google.cloud.pubsub.v1.SubscriptionAdminClient;
import com.google.cloud.pubsub.v1.SubscriptionAdminSettings;
import com.google.cloud.pubsub.v1.TopicAdminClient;
import com.google.cloud.pubsub.v1.TopicAdminSettings;
import com.google.protobuf.Duration;
import com.google.pubsub.v1.ListSubscriptionsRequest;
import com.google.pubsub.v1.ListTopicsRequest;
import com.google.pubsub.v1.ProjectName;
import com.google.pubsub.v1.ProjectSubscriptionName;
import com.google.pubsub.v1.ProjectTopicName;
import com.google.pubsub.v1.PushConfig;
import com.google.pubsub.v1.Subscription;
import com.google.pubsub.v1.Topic;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.queue.TbQueueAdmin;
import java.io.IOException;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@Slf4j
public class TbPubSubAdmin implements TbQueueAdmin {
private static final String ACK_DEADLINE = "ackDeadlineInSec";
private static final String MESSAGE_RETENTION = "messageRetentionInSec";
private final TopicAdminClient topicAdminClient;
private final SubscriptionAdminClient subscriptionAdminClient;
private final TbPubSubSettings pubSubSettings;
private final SubscriptionAdminSettings subscriptionAdminSettings;
private final TopicAdminSettings topicAdminSettings;
private final Set<String> topicSet = ConcurrentHashMap.newKeySet();
private final Set<String> subscriptionSet = ConcurrentHashMap.newKeySet();
private final Map<String, String> subscriptionProperties;
public TbPubSubAdmin(TbPubSubSettings pubSubSettings) {
public TbPubSubAdmin(TbPubSubSettings pubSubSettings, Map<String, String> subscriptionSettings) {
this.pubSubSettings = pubSubSettings;
this.subscriptionProperties = subscriptionSettings;
TopicAdminSettings topicAdminSettings;
try {
topicAdminSettings = TopicAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build();
} catch (IOException e) {
@ -53,6 +60,7 @@ public class TbPubSubAdmin implements TbQueueAdmin {
throw new RuntimeException("Failed to create TopicAdminSettings.");
}
SubscriptionAdminSettings subscriptionAdminSettings;
try {
subscriptionAdminSettings = SubscriptionAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build();
} catch (IOException e) {
@ -60,7 +68,9 @@ public class TbPubSubAdmin implements TbQueueAdmin {
throw new RuntimeException("Failed to create SubscriptionAdminSettings.");
}
try (TopicAdminClient topicAdminClient = TopicAdminClient.create(topicAdminSettings)) {
try {
topicAdminClient = TopicAdminClient.create(topicAdminSettings);
ListTopicsRequest listTopicsRequest =
ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build();
TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest);
@ -72,7 +82,8 @@ public class TbPubSubAdmin implements TbQueueAdmin {
throw new RuntimeException("Failed to get topics.", e);
}
try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings)) {
try {
subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings);
ListSubscriptionsRequest listSubscriptionsRequest =
ListSubscriptionsRequest.newBuilder()
@ -99,31 +110,21 @@ public class TbPubSubAdmin implements TbQueueAdmin {
return;
}
try (TopicAdminClient topicAdminClient = TopicAdminClient.create(topicAdminSettings)) {
ListTopicsRequest listTopicsRequest =
ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build();
TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest);
for (Topic topic : response.iterateAll()) {
if (topic.getName().contains(topicName.toString())) {
topicSet.add(topic.getName());
createSubscriptionIfNotExists(partition, topicName);
return;
}
ListTopicsRequest listTopicsRequest =
ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build();
TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest);
for (Topic topic : response.iterateAll()) {
if (topic.getName().contains(topicName.toString())) {
topicSet.add(topic.getName());
createSubscriptionIfNotExists(partition, topicName);
return;
}
topicAdminClient.createTopic(topicName);
topicSet.add(topicName.toString());
log.info("Created new topic: [{}]", topicName.toString());
createSubscriptionIfNotExists(partition, topicName);
} catch (IOException e) {
log.error("Failed to create topic: [{}].", topicName.toString(), e);
throw new RuntimeException("Failed to create topic.", e);
}
}
@Override
public void destroy() {
topicAdminClient.createTopic(topicName);
topicSet.add(topicName.toString());
log.info("Created new topic: [{}]", topicName.toString());
createSubscriptionIfNotExists(partition, topicName);
}
private void createSubscriptionIfNotExists(String partition, ProjectTopicName topicName) {
@ -134,29 +135,52 @@ public class TbPubSubAdmin implements TbQueueAdmin {
return;
}
try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings)) {
ListSubscriptionsRequest listSubscriptionsRequest =
ListSubscriptionsRequest.newBuilder()
.setProject(ProjectName.of(pubSubSettings.getProjectId()).toString())
.build();
SubscriptionAdminClient.ListSubscriptionsPagedResponse response =
subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest);
for (Subscription subscription : response.iterateAll()) {
if (subscription.getName().equals(subscriptionName.toString())) {
subscriptionSet.add(subscription.getName());
return;
}
ListSubscriptionsRequest listSubscriptionsRequest =
ListSubscriptionsRequest.newBuilder().setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()).build();
SubscriptionAdminClient.ListSubscriptionsPagedResponse response = subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest);
for (Subscription subscription : response.iterateAll()) {
if (subscription.getName().equals(subscriptionName.toString())) {
subscriptionSet.add(subscription.getName());
return;
}
}
subscriptionAdminClient.createSubscription(
subscriptionName, topicName, PushConfig.getDefaultInstance(), pubSubSettings.getAckDeadline()).getName();
subscriptionSet.add(subscriptionName.toString());
log.info("Created new subscription: [{}]", subscriptionName.toString());
} catch (IOException e) {
log.error("Failed to create subscription: [{}].", subscriptionName.toString(), e);
throw new RuntimeException("Failed to create subscription.", e);
Subscription.Builder subscriptionBuilder = Subscription
.newBuilder()
.setName(subscriptionName.toString())
.setTopic(topicName.toString());
setAckDeadline(subscriptionBuilder);
setMessageRetention(subscriptionBuilder);
subscriptionAdminClient.createSubscription(subscriptionBuilder.build());
subscriptionSet.add(subscriptionName.toString());
log.info("Created new subscription: [{}]", subscriptionName.toString());
}
private void setAckDeadline(Subscription.Builder builder) {
if (subscriptionProperties.containsKey(ACK_DEADLINE)) {
builder.setAckDeadlineSeconds(Integer.parseInt(subscriptionProperties.get(ACK_DEADLINE)));
}
}
private void setMessageRetention(Subscription.Builder builder) {
if (subscriptionProperties.containsKey(MESSAGE_RETENTION)) {
Duration duration = Duration
.newBuilder()
.setSeconds(Long.parseLong(subscriptionProperties.get(MESSAGE_RETENTION)))
.build();
builder.setMessageRetentionDuration(duration);
}
}
@Override
public void destroy() {
if (topicAdminClient != null) {
topicAdminClient.close();
}
if (subscriptionAdminClient != null) {
subscriptionAdminClient.close();
}
}
}

3
common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java
View File

@ -40,9 +40,6 @@ public class TbPubSubSettings {
@Value("${queue.pubsub.service_account}")
private String serviceAccount;
@Value("${queue.pubsub.ack_deadline}")
private int ackDeadline;
@Value("${queue.pubsub.max_msg_size}")
private int maxMsgSize;

71
common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java
View File

@ -0,0 +1,71 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.pubsub;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.HashMap;
import java.util.Map;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='pubsub'")
public class TbPubSubSubscriptionSettings {
@Value("${queue.pubsub.queue-properties.core}")
private String coreProperties;
@Value("${queue.pubsub.queue-properties.rule-engine}")
private String ruleEngineProperties;
@Value("${queue.pubsub.queue-properties.transport-api}")
private String transportApiProperties;
@Value("${queue.pubsub.queue-properties.notifications}")
private String notificationsProperties;
@Value("${queue.pubsub.queue-properties.js-executor}")
private String jsExecutorProperties;
@Getter
private Map<String, String> coreSettings;
@Getter
private Map<String, String> ruleEngineSettings;
@Getter
private Map<String, String> transportApiSettings;
@Getter
private Map<String, String> notificationsSettings;
@Getter
private Map<String, String> jsExecutorSettings;
@PostConstruct
private void init() {
coreSettings = getSettings(coreProperties);
ruleEngineSettings = getSettings(ruleEngineProperties);
transportApiSettings = getSettings(transportApiProperties);
notificationsSettings = getSettings(notificationsProperties);
jsExecutorSettings = getSettings(jsExecutorProperties);
}
private Map<String, String> getSettings(String properties) {
Map<String, String> configs = new HashMap<>();
for (String property : properties.split(";")) {
int delimiterPosition = property.indexOf(":");
String key = property.substring(0, delimiterPosition);
String value = property.substring(delimiterPosition + 1);
configs.put(key, value);
}
return configs;
}
}

11
common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java
View File

@ -18,24 +18,23 @@ package org.thingsboard.server.queue.rabbitmq;
import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Connection;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Component;
import org.thingsboard.server.queue.TbQueueAdmin;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.TimeoutException;
@Slf4j
@Component
@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'")
public class TbRabbitMqAdmin implements TbQueueAdmin {
private final TbRabbitMqSettings rabbitMqSettings;
private final Channel channel;
private final Connection connection;
private final Map<String, Object> arguments;
public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings) {
public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings, Map<String, Object> arguments) {
this.rabbitMqSettings = rabbitMqSettings;
this.arguments = arguments;
try {
connection = rabbitMqSettings.getConnectionFactory().newConnection();
@ -55,7 +54,7 @@ public class TbRabbitMqAdmin implements TbQueueAdmin {
@Override
public void createTopicIfNotExists(String topic) {
try {
channel.queueDeclare(topic, false, false, false, null);
channel.queueDeclare(topic, false, false, false, arguments);
} catch (IOException e) {
log.error("Failed to bind queue: [{}]", topic, e);
}

98
common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java
View File

@ -0,0 +1,98 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.rabbitmq;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'")
public class TbRabbitMqQueueArguments {
@Value("${queue.rabbitmq.queue-properties.core}")
private String coreProperties;
@Value("${queue.rabbitmq.queue-properties.rule-engine}")
private String ruleEngineProperties;
@Value("${queue.rabbitmq.queue-properties.transport-api}")
private String transportApiProperties;
@Value("${queue.rabbitmq.queue-properties.notifications}")
private String notificationsProperties;
@Value("${queue.rabbitmq.queue-properties.js-executor}")
private String jsExecutorProperties;
@Getter
private Map<String, Object> coreArgs;
@Getter
private Map<String, Object> ruleEngineArgs;
@Getter
private Map<String, Object> transportApiArgs;
@Getter
private Map<String, Object> notificationsArgs;
@Getter
private Map<String, Object> jsExecutorArgs;
@PostConstruct
private void init() {
coreArgs = getArgs(coreProperties);
ruleEngineArgs = getArgs(ruleEngineProperties);
transportApiArgs = getArgs(transportApiProperties);
notificationsArgs = getArgs(notificationsProperties);
jsExecutorArgs = getArgs(jsExecutorProperties);
}
private Map<String, Object> getArgs(String properties) {
Map<String, Object> configs = new HashMap<>();
for (String property : properties.split(";")) {
int delimiterPosition = property.indexOf(":");
String key = property.substring(0, delimiterPosition);
String strValue = property.substring(delimiterPosition + 1);
configs.put(key, getObjectValue(strValue));
}
return configs;
}
private Object getObjectValue(String str) {
if (str.equalsIgnoreCase("true") || str.equalsIgnoreCase("false")) {
return Boolean.valueOf(str);
} else if (isNumeric(str)) {
return getNumericValue(str);
}
return str;
}
private Object getNumericValue(String str) {
if (str.contains(".")) {
return Double.valueOf(str);
} else {
return Long.valueOf(str);
}
}
private static final Pattern PATTERN = Pattern.compile("-?\\d+(\\.\\d+)?");
public boolean isNumeric(String strNum) {
if (strNum == null) {
return false;
}
return PATTERN.matcher(strNum).matches();
}
}

2
common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java
View File

@ -24,7 +24,7 @@ public class TbRuleEngineQueueConfiguration {
private String topic;
private int pollInterval;
private int partitions;
private String packProcessingTimeout;
private long packProcessingTimeout;
private TbRuleEngineQueueSubmitStrategyConfiguration submitStrategy;
private TbRuleEngineQueueAckStrategyConfiguration processingStrategy;

12
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java
View File

@ -138,12 +138,24 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
break;
case PUBLISH:
processPublish(ctx, (MqttPublishMessage) msg);
transportService.reportActivity(sessionInfo);
if (gatewaySessionHandler != null) {
gatewaySessionHandler.reportActivity();
}
break;
case SUBSCRIBE:
processSubscribe(ctx, (MqttSubscribeMessage) msg);
transportService.reportActivity(sessionInfo);
if (gatewaySessionHandler != null) {
gatewaySessionHandler.reportActivity();
}
break;
case UNSUBSCRIBE:
processUnsubscribe(ctx, (MqttUnsubscribeMessage) msg);
transportService.reportActivity(sessionInfo);
if (gatewaySessionHandler != null) {
gatewaySessionHandler.reportActivity();
}
break;
case PINGREQ:
if (checkConnected(ctx, msg)) {

8
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java
View File

@ -544,6 +544,9 @@ public class DefaultTransportService implements TransportService {
protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback<Void> callback) {
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, getTenantId(sessionInfo), getDeviceId(sessionInfo));
if (log.isTraceEnabled()) {
log.trace("[{}][{}] Pushing to topic {} message {}", getTenantId(sessionInfo), getDeviceId(sessionInfo), tpi.getFullTopicName(), toDeviceActorMsg);
}
tbCoreMsgProducer.send(tpi,
new TbProtoQueueMsg<>(getRoutingKey(sessionInfo),
ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ?
@ -552,6 +555,9 @@ public class DefaultTransportService implements TransportService {
protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) {
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator());
if (log.isTraceEnabled()) {
log.trace("[{}][{}] Pushing to topic {} message {}", tenantId, tbMsg.getOriginator(), tpi.getFullTopicName(), tbMsg);
}
ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg))
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build();
@ -588,7 +594,7 @@ public class DefaultTransportService implements TransportService {
@Override
public void onSuccess(TbQueueMsgMetadata metadata) {
if (msgCount.decrementAndGet() <= 0) {
callback.onSuccess(null);
DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onSuccess(null));
}
}

3
dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java
View File

@ -32,6 +32,9 @@ public class ModelConstants {
public static final String NULL_UUID_STR = UUIDConverter.fromTimeUUID(NULL_UUID);
public static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID);
// this is the difference between midnight October 15, 1582 UTC and midnight January 1, 1970 UTC as 100 nanosecond units
public static final long EPOCH_DIFF = 122192928000000000L;
/**
* Generic constants.
*/

14
dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java
View File

@ -37,6 +37,9 @@ import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.Table;
import java.util.UUID;
import static org.thingsboard.server.dao.model.ModelConstants.EPOCH_DIFF;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_BODY_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_COLUMN_FAMILY_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_PROPERTY;
@ -44,6 +47,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TENANT_ID_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_UID_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN;
@Data
@EqualsAndHashCode(callSuper = true)
@ -73,9 +77,15 @@ public class EventEntity extends BaseSqlEntity<Event> implements BaseEntity<Eve
@Column(name = EVENT_BODY_PROPERTY)
private JsonNode body;
@Column(name = TS_COLUMN)
private long ts;
public EventEntity(Event event) {
if (event.getId() != null) {
this.setUuid(event.getId().getId());
this.ts = getTs(event.getId().getId());
} else {
this.ts = System.currentTimeMillis();
}
if (event.getTenantId() != null) {
this.tenantId = toString(event.getTenantId().getId());
@ -101,4 +111,8 @@ public class EventEntity extends BaseSqlEntity<Event> implements BaseEntity<Eve
event.setUid(eventUid);
return event;
}
private long getTs(UUID uuid) {
return (uuid.timestamp() - EPOCH_DIFF) / 10000;
}
}

25
dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java
View File

@ -88,26 +88,33 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC
RuleChain ruleChain = ruleChainDao.findById(tenantId, ruleChainId.getId());
if (!ruleChain.isRoot()) {
RuleChain previousRootRuleChain = getRootTenantRuleChain(ruleChain.getTenantId());
if (!previousRootRuleChain.getId().equals(ruleChain.getId())) {
try {
try {
if (previousRootRuleChain == null) {
setRootAndSave(tenantId, ruleChain);
return true;
} else if (!previousRootRuleChain.getId().equals(ruleChain.getId())) {
deleteRelation(tenantId, new EntityRelation(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(),
EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN));
previousRootRuleChain.setRoot(false);
ruleChainDao.save(tenantId, previousRootRuleChain);
createRelation(tenantId, new EntityRelation(ruleChain.getTenantId(), ruleChain.getId(),
EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN));
ruleChain.setRoot(true);
ruleChainDao.save(tenantId, ruleChain);
setRootAndSave(tenantId, ruleChain);
return true;
} catch (ExecutionException | InterruptedException e) {
log.warn("[{}] Failed to set root rule chain, ruleChainId: [{}]", ruleChainId);
throw new RuntimeException(e);
}
} catch (ExecutionException | InterruptedException e) {
log.warn("[{}] Failed to set root rule chain, ruleChainId: [{}]", ruleChainId);
throw new RuntimeException(e);
}
}
return false;
}
private void setRootAndSave(TenantId tenantId, RuleChain ruleChain) throws ExecutionException, InterruptedException {
createRelation(tenantId, new EntityRelation(ruleChain.getTenantId(), ruleChain.getId(),
EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN));
ruleChain.setRoot(true);
ruleChainDao.save(tenantId, ruleChain);
}
@Override
public RuleChainMetaData saveRuleChainMetaData(TenantId tenantId, RuleChainMetaData ruleChainMetaData) {
Validator.validateId(ruleChainMetaData.getRuleChainId(), "Incorrect rule chain id.");

3
dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java
View File

@ -75,7 +75,8 @@ public abstract class AbstractEventInsertRepository implements EventInsertReposi
.setParameter("entity_type", entity.getEntityType().name())
.setParameter("event_type", entity.getEventType())
.setParameter("event_uid", entity.getEventUid())
.setParameter("tenant_id", entity.getTenantId());
.setParameter("tenant_id", entity.getTenantId())
.setParameter("ts", entity.getTs());
}
private EventEntity processSaveOrUpdate(EventEntity entity, String query) {

4
dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java
View File

@ -44,7 +44,7 @@ public class HsqlEventInsertRepository extends AbstractEventInsertRepository {
}
private static String getInsertString(String conflictStatement) {
return "MERGE INTO event USING (VALUES :id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id) I (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) ON " + conflictStatement + " WHEN MATCHED THEN UPDATE SET event.id = I.id, event.body = I.body, event.entity_id = I.entity_id, event.entity_type = I.entity_type, event.event_type = I.event_type, event.event_uid = I.event_uid, event.tenant_id = I.tenant_id" +
" WHEN NOT MATCHED THEN INSERT (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) VALUES (I.id, I.body, I.entity_id, I.entity_type, I.event_type, I.event_uid, I.tenant_id)";
return "MERGE INTO event USING (VALUES :id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id, :ts) I (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) ON " + conflictStatement + " WHEN MATCHED THEN UPDATE SET event.id = I.id, event.body = I.body, event.entity_id = I.entity_id, event.entity_type = I.entity_type, event.event_type = I.event_type, event.event_uid = I.event_uid, event.tenant_id = I.tenant_id, event.ts = I.ts" +
" WHEN NOT MATCHED THEN INSERT (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) VALUES (I.id, I.body, I.entity_id, I.entity_type, I.event_type, I.event_uid, I.tenant_id, I.ts)";
}
}

2
dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java
View File

@ -48,6 +48,6 @@ public class PsqlEventInsertRepository extends AbstractEventInsertRepository {
}
private static String getInsertOrUpdateString(String eventKeyStatement, String updateKeyStatement) {
return "INSERT INTO event (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) VALUES (:id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id) ON CONFLICT " + eventKeyStatement + " DO UPDATE SET body = :body, " + updateKeyStatement + " returning *";
return "INSERT INTO event (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) VALUES (:id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id, :ts) ON CONFLICT " + eventKeyStatement + " DO UPDATE SET body = :body, ts = :ts," + updateKeyStatement + " returning *";
}
}

2
dao/src/main/resources/sql/schema-entities-hsql.sql
View File

@ -144,6 +144,7 @@ CREATE TABLE IF NOT EXISTS event (
event_type varchar(255),
event_uid varchar(255),
tenant_id varchar(31),
ts bigint NOT NULL,
CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid)
);
@ -251,3 +252,4 @@ CREATE TABLE IF NOT EXISTS entity_view (
search_text varchar(255),
additional_info varchar
);

26
dao/src/main/resources/sql/schema-entities.sql
View File

@ -144,6 +144,7 @@ CREATE TABLE IF NOT EXISTS event (
event_type varchar(255),
event_uid varchar(255),
tenant_id varchar(31),
ts bigint NOT NULL,
CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid)
);
@ -251,3 +252,28 @@ CREATE TABLE IF NOT EXISTS entity_view (
search_text varchar(255),
additional_info varchar
);
CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint)
LANGUAGE plpgsql AS
$$
DECLARE
ttl_ts bigint;
debug_ttl_ts bigint;
ttl_deleted_count bigint DEFAULT 0;
debug_ttl_deleted_count bigint DEFAULT 0;
BEGIN
IF ttl > 0 THEN
ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint;
EXECUTE format(
'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count;
END IF;
IF debug_ttl > 0 THEN
debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint;
EXECUTE format(
'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count;
END IF;
RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count;
RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count;
deleted := ttl_deleted_count + debug_ttl_deleted_count;
END
$$;

2
dao/src/main/resources/sql/schema-timescale.sql
View File

@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings
CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version)
);
INSERT INTO tb_schema_settings (schema_version) VALUES (2005000);
INSERT INTO tb_schema_settings (schema_version) VALUES (2005000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005000;
CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS
$$

2
dao/src/main/resources/sql/schema-ts-psql.sql
View File

@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings
CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version)
);
INSERT INTO tb_schema_settings (schema_version) VALUES (2005000);
INSERT INTO tb_schema_settings (schema_version) VALUES (2005000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005000;
CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint)
LANGUAGE plpgsql AS

Some files were not shown because too many files changed in this diff

Write Preview
Loading…
Cancel
Save